diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..cb4c23e4e0b2e1d12140576d37416502d8738522 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,34 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 filter=lfs diff=lfs merge=lfs -text +hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 filter=lfs diff=lfs merge=lfs -text +hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 filter=lfs diff=lfs merge=lfs -text +hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 filter=lfs diff=lfs merge=lfs -text +hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 filter=lfs diff=lfs merge=lfs -text +hab6e0c2/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hab6e0c2/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hab6e0c2/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hab6e0c2/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hab6e0c2/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +4k07t1xl/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qo464509/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7krw5tkq/episode_rosbags/episode_4_2025-01-01-11-33-45.bag filter=lfs diff=lfs merge=lfs -text diff --git a/4k07t1xl/.hydra/config.yaml b/4k07t1xl/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/4k07t1xl/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/4k07t1xl/.hydra/hydra.yaml b/4k07t1xl/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22622ebfc81bab1c67512c9a4527333162d1a817 --- /dev/null +++ b/4k07t1xl/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/4k07t1xl/.hydra/overrides.yaml b/4k07t1xl/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/4k07t1xl/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/4k07t1xl/episode_rosbags/aligned_depth_to_color_K.npy b/4k07t1xl/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/4k07t1xl/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/4k07t1xl/episode_rosbags/cam_tf_world.npy b/4k07t1xl/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/4k07t1xl/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/4k07t1xl/episode_rosbags/color_K.npy b/4k07t1xl/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/4k07t1xl/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/4k07t1xl/episode_rosbags/depth_K.npy b/4k07t1xl/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/4k07t1xl/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/4k07t1xl/eval_robot.log b/4k07t1xl/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..71317e650a47a2b8464d61e8f5fd7cdd46c0c28e --- /dev/null +++ b/4k07t1xl/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 11:34:45,998][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:34:46,000][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/4k07t1xl/eval_video/0_eval.mp4 b/4k07t1xl/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f71263de192e738267fa7ff460c90c4ff4fd2dda --- /dev/null +++ b/4k07t1xl/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0def80a523fcf0f756011882fd1834f31c8bbbd1dda82834e01408b874db2c5 +size 1114521 diff --git a/4k07t1xl/eval_video/1_eval.mp4 b/4k07t1xl/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2627c37aabdbfaed5f19746947a7e671e9100155 --- /dev/null +++ b/4k07t1xl/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:506a301f481b1b43d783f0b1729c200967015613cb8c856d8cf24bf5f28ad82c +size 1094264 diff --git a/4k07t1xl/eval_video/2_eval.mp4 b/4k07t1xl/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..26e76d58561a54b98357b28fe6d27d2a6295e154 --- /dev/null +++ b/4k07t1xl/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1267f74013e09ddf74b1ce386eab8874b7c3ee631cbe123b20883233c218e800 +size 1102081 diff --git a/4k07t1xl/eval_video/3_eval.mp4 b/4k07t1xl/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7de66e8d7db9860fccb2ce362488cef8d058e318 --- /dev/null +++ b/4k07t1xl/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3882f24ae83d74169feb1e0493e723f6ebaa6fb560c0fec5e726e5e633db37f +size 1142341 diff --git a/4k07t1xl/eval_video/4_eval.mp4 b/4k07t1xl/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca1930cb28f9e528116388d733d970cf184bd097 --- /dev/null +++ b/4k07t1xl/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50280c95fd108de973aa051d15fbb1a2aa9c77df14a27e53f640fcf41ae576bf +size 1134352 diff --git a/4k07t1xl/tb/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 b/4k07t1xl/tb/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 new file mode 100644 index 0000000000000000000000000000000000000000..93a11f71f23d1a346fb51c5cbd71fb8b48983d6e --- /dev/null +++ b/4k07t1xl/tb/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5add61ea3e7fe98d864794c7a870a67480044a422693d31d0c3cade42e48942 +size 6539 diff --git a/4k07t1xl/wandb/debug-internal.log b/4k07t1xl/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b3f3e35140152fbea2c3fb6663659e1a6cedc56e --- /dev/null +++ b/4k07t1xl/wandb/debug-internal.log @@ -0,0 +1,892 @@ +2025-01-01 11:34:51,744 INFO StreamThr :1795590 [internal.py:wandb_internal():85] W&B internal server running at pid: 1795590, started at: 2025-01-01 11:34:51.744537 +2025-01-01 11:34:51,746 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:34:51,757 INFO WriterThread:1795590 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/run-4k07t1xl.wandb +2025-01-01 11:34:51,758 DEBUG SenderThread:1795590 [sender.py:send():379] send: header +2025-01-01 11:34:51,764 DEBUG SenderThread:1795590 [sender.py:send():379] send: run +2025-01-01 11:34:52,022 INFO SenderThread:1795590 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files +2025-01-01 11:34:52,022 INFO SenderThread:1795590 [sender.py:_start_run_threads():1188] run started: 4k07t1xl with start time 1735749291.745044 +2025-01-01 11:34:52,038 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:34:52,038 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:34:52,113 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:34:52,118 DEBUG HandlerThread:1795590 [system_info.py:__init__():26] System info init +2025-01-01 11:34:52,118 DEBUG HandlerThread:1795590 [system_info.py:__init__():41] System info init done +2025-01-01 11:34:52,118 INFO HandlerThread:1795590 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:34:52,118 INFO SystemMonitor:1795590 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:34:52,119 INFO HandlerThread:1795590 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:34:52,119 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:34:52,120 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:34:52,120 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:34:52,121 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:34:52,121 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:34:52,130 DEBUG HandlerThread:1795590 [system_info.py:probe():152] Probing system +2025-01-01 11:34:52,132 DEBUG HandlerThread:1795590 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:34:52,140 DEBUG HandlerThread:1795590 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:34:52,140 DEBUG HandlerThread:1795590 [system_info.py:probe():200] Probing system done +2025-01-01 11:34:52,140 DEBUG HandlerThread:1795590 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:34:52.130976', 'startedAt': '2025-01-01T16:34:51.736821', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.949, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4494.85, 'min': 800.0, 'max': 4700.0}, {'current': 4498.808, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 710.8477401733398}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:34:52,140 INFO HandlerThread:1795590 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:34:52,140 INFO HandlerThread:1795590 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:34:52,140 DEBUG HandlerThread:1795590 [system_info.py:_save_code():44] Saving code +2025-01-01 11:34:52,142 DEBUG HandlerThread:1795590 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:34:52,142 DEBUG HandlerThread:1795590 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:34:52,204 DEBUG HandlerThread:1795590 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:34:52,204 INFO HandlerThread:1795590 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:34:52,208 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:34:52,208 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:34:52,208 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:34:52,209 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:34:52,459 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:34:52,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:34:52,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:34:52,461 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:34:52,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:52,462 INFO HandlerThread:1795590 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/tb" + save: true +} +control { + relay_id: "8569b3a4bd464a76b8bdd8443aa0b49c" +} +_info { + stream_id: "4k07t1xl" +} + +2025-01-01 11:34:52,463 DEBUG HandlerThread:1795590 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:34:52,504 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/j7w6apqv-wandb-metadata.json +2025-01-01 11:34:52,516 INFO wandb-upload_2:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/n06ek16b-diff.patch +2025-01-01 11:34:52,519 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/ulu6svnf-code/FISH/eval_robot.py +2025-01-01 11:34:52,601 DEBUG SenderThread:1795590 [sender.py:send():379] send: telemetry +2025-01-01 11:34:52,601 DEBUG SenderThread:1795590 [sender.py:send():379] send: tbrecord +2025-01-01 11:34:52,601 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:34:52,601 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 with policy live +2025-01-01 11:34:52,856 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/q3984jc4-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:34:53,023 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code/FISH/eval_robot.py +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/diff.patch +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/requirements.txt +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-metadata.json +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code/FISH +2025-01-01 11:34:53,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:54,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:55,023 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:34:55,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:56,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:56,762 DEBUG SenderThread:1795590 [sender.py:send():379] send: config +2025-01-01 11:34:56,762 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:57,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:58,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:59,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:34:59,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:00,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:01,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:01,773 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:02,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:03,025 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:03,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:04,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:05,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:06,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:06,873 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:07,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:07,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:07,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:08,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:09,027 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:09,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:10,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:11,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:12,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:12,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:13,028 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:13,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:14,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:15,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:16,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:17,029 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:17,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:17,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:18,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:19,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:20,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:21,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:22,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:22,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:22,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:22,612 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:23,031 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml +2025-01-01 11:35:23,031 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:23,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:24,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:25,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:26,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:27,032 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:27,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:27,897 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:28,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:29,032 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:29,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:30,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:31,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:32,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:32,904 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:33,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:34,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:35,034 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:35,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:36,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:37,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:37,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:37,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:38,004 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:38,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:39,035 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:39,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:40,469 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:41,036 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:41,469 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:42,477 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:43,037 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:43,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:43,480 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:44,478 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:45,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:46,478 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:47,478 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:48,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:49,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:49,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:50,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:51,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:52,121 DEBUG SystemMonitor:1795590 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:35:52,122 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:35:52,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:52,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:52,501 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:53,486 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:54,486 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:54,605 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:55,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:56,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:57,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:58,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:59,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:00,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:00,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:01,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:02,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:03,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:04,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:05,042 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:05,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:05,778 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:06,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:07,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:07,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:07,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:08,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:09,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:09,697 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:36:09,698 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:36:09,698 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 with policy now +2025-01-01 11:36:09,698 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:36:09,698 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:36:09,699 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:36:09,913 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/gai242ym-media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos +2025-01-01 11:36:10,280 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/gdnq6obg-media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 +2025-01-01 11:36:10,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:11,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:11,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:11,699 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:12,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:13,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:14,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:15,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:16,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:16,725 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:17,046 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:17,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:18,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:19,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:20,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:21,047 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:21,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:21,829 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:22,122 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:36:22,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:22,464 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:22,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:23,048 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:23,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:24,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:25,049 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:25,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:26,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:27,050 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:27,050 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:36:27,243 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:27,298 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/vj3z41ys-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:36:27,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:28,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:29,050 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:29,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:30,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:31,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:32,253 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:32,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:33,052 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:33,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:34,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:35,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:36,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:37,312 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:37,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:37,461 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:37,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:38,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:39,054 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:39,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:40,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:41,054 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:41,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:42,471 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:42,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:43,055 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:43,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:44,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:45,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:46,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:47,471 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:47,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:48,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:49,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:50,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:51,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:52,123 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:36:52,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:52,461 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:52,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:52,525 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:53,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:54,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:55,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:56,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:57,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:58,472 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:58,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:59,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:00,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:01,497 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:02,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:03,472 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:03,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:04,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:05,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:06,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:07,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:07,462 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:07,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:08,268 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:37:08,268 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 with policy now +2025-01-01 11:37:08,271 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:37:08,271 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:37:08,271 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:37:08,273 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:37:08,476 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/p2o572sw-media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 +2025-01-01 11:37:08,497 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:09,063 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:37:09,063 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:09,064 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 +2025-01-01 11:37:09,064 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:37:09,285 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:09,317 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/715folg8-media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 +2025-01-01 11:37:09,497 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:10,499 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:11,064 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:11,500 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:12,500 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:13,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:14,333 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:14,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:15,065 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:15,501 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:16,502 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:17,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:18,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:19,398 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:19,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:20,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:21,067 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:21,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:22,123 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:37:22,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:22,462 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:22,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:23,068 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:23,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:24,068 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:37:24,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:24,859 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:24,924 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:37:24,924 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:37:24,925 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:37:25,069 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:37:25,069 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:25,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:26,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:27,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:28,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:29,504 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:29,889 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:30,504 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:31,071 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:31,504 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:32,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:33,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:34,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:34,951 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:35,072 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:35,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:36,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:37,073 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:37,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:37,462 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:37,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:38,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:39,074 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:39,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:40,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:40,902 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:41,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:42,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:43,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:44,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:45,076 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:45,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:46,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:46,694 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:47,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:48,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:49,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:50,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:51,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:52,124 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:37:52,124 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:52,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:52,462 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:52,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:53,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:54,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:55,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:56,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:57,477 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:57,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:58,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:59,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:00,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:01,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:02,477 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:02,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:03,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:04,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:05,512 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:05,981 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:38:05,982 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:38:05,982 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 with policy now +2025-01-01 11:38:05,982 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:38:05,982 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:38:05,983 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:38:06,087 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:38:06,087 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 +2025-01-01 11:38:06,087 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:38:06,186 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/mm7fzv1s-media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 +2025-01-01 11:38:06,429 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/q5yk7gbg-media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 +2025-01-01 11:38:06,515 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:07,087 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:07,472 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:07,472 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:07,514 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:07,555 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:08,088 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:08,515 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:09,517 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:10,516 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:11,517 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:12,518 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:13,089 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:13,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:13,520 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:14,526 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:15,525 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:16,521 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:17,091 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:17,523 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:18,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:18,524 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:19,092 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:19,525 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:20,520 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:21,092 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:21,523 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:22,124 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:38:22,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:22,468 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:22,521 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:23,524 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:23,620 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:24,527 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:25,530 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:26,094 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:38:26,323 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/6hmvm48u-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:38:26,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:26,777 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:38:26,777 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:38:26,777 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:38:27,094 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:38:27,095 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:27,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:28,526 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:29,480 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:29,528 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:30,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:31,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:32,530 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:33,096 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:33,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:34,481 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:34,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:35,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:36,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:37,098 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:37,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:37,464 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:37,530 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:38,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:39,098 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:39,523 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:39,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:40,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:41,099 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:41,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:42,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:43,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:44,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:45,482 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:45,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:46,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:47,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:48,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:49,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:50,482 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:50,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:51,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:52,124 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:38:52,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:52,464 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:52,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:53,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:54,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:55,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:55,616 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:56,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:57,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:58,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:59,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:00,534 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:01,482 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:01,534 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:02,534 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:03,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:04,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:05,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:06,483 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:06,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:07,468 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:07,468 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:07,536 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:08,404 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:39:08,404 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 with policy now +2025-01-01 11:39:08,405 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:39:08,407 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:39:08,407 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:39:08,407 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:39:08,543 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:08,641 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/1mg1d2yl-media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 +2025-01-01 11:39:09,108 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:39:09,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:09,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 +2025-01-01 11:39:09,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:39:09,371 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/sv7kqvuk-media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 +2025-01-01 11:39:09,547 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:10,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:10,545 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:11,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:11,545 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:12,447 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:12,546 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:13,546 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:14,549 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:15,110 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:15,557 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:16,548 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:17,486 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:17,554 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:18,593 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:19,559 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:20,557 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:21,112 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:21,556 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:22,125 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:39:22,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:22,465 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:22,554 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:22,581 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:23,113 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:23,555 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:24,556 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:25,558 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:26,558 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:27,558 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:28,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:28,564 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:29,562 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:30,560 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:31,569 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:32,568 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:33,571 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:33,856 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:39:33,856 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:39:33,856 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:33,857 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:39:34,117 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:39:34,117 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:39:34,345 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/77h6cx4p-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:39:34,562 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:35,117 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:35,564 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:36,567 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:37,469 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:37,469 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:37,566 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:38,572 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:39,118 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:39,373 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:39,568 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:40,568 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:41,569 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:42,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:43,571 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:44,437 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:44,573 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:45,120 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:45,575 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:46,572 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:47,121 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:47,593 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:48,575 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:49,122 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:49,575 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:50,375 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:50,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:51,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:52,125 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:39:52,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:52,465 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:52,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:53,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:54,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:55,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:55,604 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:56,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:57,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:58,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:59,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:00,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:01,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:01,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:02,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:03,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:04,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:05,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:06,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:06,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:07,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:40:07,465 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:40:07,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:08,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:09,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:10,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:11,555 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:11,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:12,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:13,580 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:14,580 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:15,130 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:15,586 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:15,725 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:40:15,725 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:40:15,725 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 with policy now +2025-01-01 11:40:15,726 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:15,726 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:15,726 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:15,930 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/xjtbx3fk-media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 +2025-01-01 11:40:16,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:40:16,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 +2025-01-01 11:40:16,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:40:16,368 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/pglqyh5i-media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 +2025-01-01 11:40:16,582 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:16,760 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:17,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:17,591 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:18,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:18,605 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:19,588 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:20,603 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:21,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:21,809 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:22,126 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:40:22,475 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:40:22,475 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:40:22,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:23,133 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:23,601 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:24,594 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:25,602 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:26,602 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:26,875 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:27,134 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:27,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:28,600 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:29,135 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:29,599 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:30,594 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:31,136 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:31,596 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:32,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:32,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:33,137 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:40:33,525 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:40:33,525 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:40:33,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:33,743 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:40:33,936 INFO wandb-upload_0:1795590 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpo6zacjce +2025-01-01 11:40:34,136 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:40:34,137 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:40:34,137 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:40:34,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media +2025-01-01 11:40:34,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval +2025-01-01 11:40:34,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table +2025-01-01 11:40:34,145 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:40:34,292 INFO SenderThread:1795590 [sender.py:send_request_log_artifact():1525] logged artifact run-4k07t1xl-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5NjQ0MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MzU4Nw==', 'latestArtifact': None}} +2025-01-01 11:40:34,292 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:40:34,293 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 11:40:34,293 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:34,293 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,293 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,294 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:34,294 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,294 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,294 DEBUG SenderThread:1795590 [sender.py:send():379] send: telemetry +2025-01-01 11:40:34,294 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,295 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,295 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,295 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,295 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,296 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,296 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,296 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,296 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,296 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,297 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,297 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,297 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,297 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,298 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,298 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,298 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,298 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,298 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,299 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,299 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,299 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,299 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:40:34,463 INFO wandb-upload_2:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/1ffrey2x-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:40:34,596 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:34,617 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/4xojhgjr-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:40:34,703 INFO wandb-upload_2:1795590 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpt1oljlbs +2025-01-01 11:40:34,730 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/bh2etqow-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:40:35,097 INFO SenderThread:1795590 [sender.py:send_request_log_artifact():1525] logged artifact run-4k07t1xl-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5NjQ0OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MzU4OA==', 'latestArtifact': None}} +2025-01-01 11:40:35,097 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:40:35,097 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json with policy now +2025-01-01 11:40:35,097 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:35,098 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:35,098 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:35,098 DEBUG SenderThread:1795590 [sender.py:send():379] send: exit +2025-01-01 11:40:35,098 INFO SenderThread:1795590 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:40:35,099 INFO SenderThread:1795590 [sender.py:send_exit():588] handling runtime: 342 +2025-01-01 11:40:35,102 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:35,102 INFO SenderThread:1795590 [sender.py:send_exit():594] send defer +2025-01-01 11:40:35,102 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,102 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:40:35,103 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:35,103 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:40:35,103 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:40:35,103 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,103 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:40:35,103 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:35,103 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:40:35,103 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:40:35,103 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,103 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:40:35,103 INFO HandlerThread:1795590 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:40:35,103 DEBUG SystemMonitor:1795590 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:40:35,103 DEBUG SystemMonitor:1795590 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:40:35,103 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:40:35,104 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:40:35,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:40:35,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:35,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json +2025-01-01 11:40:35,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval +2025-01-01 11:40:35,158 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:40:35,159 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:40:35,159 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:40:35,159 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:35,159 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:40:35,159 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:40:35,159 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:40:35,160 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,160 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:40:35,160 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:35,160 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:40:35,160 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:40:35,160 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,160 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:40:35,311 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/3vxst9c0-media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json +2025-01-01 11:40:35,556 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/2wdrelvx-media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json +2025-01-01 11:40:37,139 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:41,934 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:41,934 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:41,935 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:40:41,935 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:41,936 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:40:41,936 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:40:41,936 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:41,936 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:41,936 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:40:41,936 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:41,937 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:41,937 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:41,937 DEBUG SenderThread:1795590 [sender.py:send():379] send: summary +2025-01-01 11:40:41,941 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:41,941 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:41,941 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:40:41,942 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:40:41,942 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:41,942 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:40:41,942 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:41,942 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:40:41,945 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:41,963 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:42,062 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:40:42,062 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:42,062 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:42,062 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:40:42,062 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:42,062 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:40:42,141 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml +2025-01-01 11:40:42,141 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:40:42,963 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:44,127 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:40:44,128 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:44,128 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:44,128 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:40:44,128 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:44,128 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:40:44,128 INFO SenderThread:1795590 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:40:44,130 INFO SenderThread:1795590 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:40:44,132 INFO SenderThread:1795590 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:40:44,142 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:44,145 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:40:44,145 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:44,145 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:40:44,146 DEBUG SenderThread:1795590 [sender.py:send():379] send: artifact +2025-01-01 11:40:44,352 INFO SenderThread:1795590 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:40:44,353 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:44,353 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:40:44,353 INFO SenderThread:1795590 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:40:44,964 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:45,142 INFO SenderThread:1795590 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml config.yaml +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json wandb-summary.json +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/requirements.txt requirements.txt +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log output.log +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/diff.patch diff.patch +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:40:45,149 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 +2025-01-01 11:40:45,149 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 +2025-01-01 11:40:45,149 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:40:45,150 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:45,150 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,150 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:40:45,151 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,151 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:40:45,151 INFO SenderThread:1795590 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:40:45,324 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:40:45,328 INFO wandb-upload_3:1795590 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml +2025-01-01 11:40:45,330 INFO wandb-upload_2:1795590 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/requirements.txt +2025-01-01 11:40:45,397 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:45,597 INFO Thread-11 (_thread_body):1795590 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:40:45,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,597 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:40:45,597 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,597 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:40:45,597 INFO SenderThread:1795590 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:40:45,599 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:40:45,599 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,599 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:40:45,599 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,599 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:40:45,599 INFO SenderThread:1795590 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:40:45,673 INFO SenderThread:1795590 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:40:45,673 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:40:45,673 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,673 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:40:45,673 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,673 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:40:45,673 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:40:45,674 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,674 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:40:45,674 DEBUG SenderThread:1795590 [sender.py:send():379] send: final +2025-01-01 11:40:45,674 DEBUG SenderThread:1795590 [sender.py:send():379] send: footer +2025-01-01 11:40:45,674 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,674 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:40:45,674 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:45,675 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:45,675 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:40:45,675 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:45,675 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:40:45,675 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:40:45,677 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:40:45,678 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:45,744 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:45,744 INFO MainThread:1795590 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:40:45,745 INFO MainThread:1795590 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:40:45,745 INFO MainThread:1795590 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:40:45,745 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:40:45,745 INFO HandlerThread:1795590 [handler.py:finish():882] shutting down handler +2025-01-01 11:40:46,676 INFO WriterThread:1795590 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/run-4k07t1xl.wandb +2025-01-01 11:40:46,744 INFO SenderThread:1795590 [sender.py:finish():1615] shutting down sender +2025-01-01 11:40:46,744 INFO SenderThread:1795590 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:40:46,744 INFO SenderThread:1795590 [file_pusher.py:join():175] waiting for file pusher diff --git a/4k07t1xl/wandb/debug.log b/4k07t1xl/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..46240777867f79a2d3b94a431a589e224f60853e --- /dev/null +++ b/4k07t1xl/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Configure stats pid to 1795426 +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/settings +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug.log +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug-internal.log +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl'} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:init():619] starting backend +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:init():623] setting up manager +2025-01-01 11:34:51,743 INFO MainThread:1795426 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:34:51,744 INFO MainThread:1795426 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:34:51,757 INFO MainThread:1795426 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:34:51,762 INFO MainThread:1795426 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:34:52,037 INFO MainThread:1795426 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:34:52,106 INFO MainThread:1795426 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:34:52,107 INFO MainThread:1795426 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:34:52,460 INFO MainThread:1795426 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:34:52,460 INFO MainThread:1795426 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:34:52,461 INFO MainThread:1795426 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:34:52,461 INFO MainThread:1795426 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:34:52,462 INFO MainThread:1795426 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:34:52,462 INFO MainThread:1795426 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/tb, True +2025-01-01 11:34:56,761 INFO MainThread:1795426 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-01 11:40:46,746 WARNING MsgRouterThr:1795426 [router.py:message_loop():77] message_loop has been closed diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code/FISH/eval_robot.py b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fe00c518d974246890e047fdf37d3f28629e6fb --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1007_0' +wandb_notes: + desc: null + value: 1007_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735749291 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/diff.patch b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..eff8de3c864ce1e6f5e6f3000cbf990769adf78e --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/diff.patch @@ -0,0 +1,105 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..b9e3dfe 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history ++# wandb_run_id: '1002_0' ++ ++# all books, 6/20 demos per book ++# RGBD+mask+act history + wandb_run_id: '1007_0' + ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c4ee39f7fc844911236fad7bcb27b1abbc17e025 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[23, 50, 0.46, 0.3002996743124774, 0.6245906352747645, 0.32984319332664913, 0.6058678477790812, 0.34994972500225807, 0.5732010900827871, 0.36585705730021656, 0.5579705722514082]]} \ No newline at end of file diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f71263de192e738267fa7ff460c90c4ff4fd2dda --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0def80a523fcf0f756011882fd1834f31c8bbbd1dda82834e01408b874db2c5 +size 1114521 diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2627c37aabdbfaed5f19746947a7e671e9100155 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:506a301f481b1b43d783f0b1729c200967015613cb8c856d8cf24bf5f28ad82c +size 1094264 diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..26e76d58561a54b98357b28fe6d27d2a6295e154 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1267f74013e09ddf74b1ce386eab8874b7c3ee631cbe123b20883233c218e800 +size 1102081 diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7de66e8d7db9860fccb2ce362488cef8d058e318 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3882f24ae83d74169feb1e0493e723f6ebaa6fb560c0fec5e726e5e633db37f +size 1142341 diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca1930cb28f9e528116388d733d970cf184bd097 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50280c95fd108de973aa051d15fbb1a2aa9c77df14a27e53f640fcf41ae576bf +size 1134352 diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..46751dba8d6eb1ecc5cec2298f8d1c27b2fc97f4 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log @@ -0,0 +1,296 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735749296.762514]: resetting environment +[INFO] [1735749296.770315]: cleared current plan +[INFO] [1735749296.771510]: moving to home +Move to target timeout: 5.000727891921997. Trans error: 0.006131293252110481. Rot error: 0.6942041875087228 +Move to target timeout: 5.099750757217407. Trans error: 0.006293444479784581. Rot error: 1.5143948887378773 +[INFO] [1735749307.874455]: reached home +[INFO] [1735749307.876035]: reset action history +[INFO] [1735749310.584159]: environment reset +Starting episode 0 +[INFO] [1735749310.588033]: resetting environment +[INFO] [1735749310.595109]: cleared current plan +[INFO] [1735749310.596775]: moving to home +Move to target timeout: 5.0006279945373535. Trans error: 0.005853875540196896. Rot error: 0.6198389996627596 +Move to target timeout: 5.100066900253296. Trans error: 0.006435464719322048. Rot error: 1.597210814663029 +[INFO] [1735749321.701291]: reached home +[INFO] [1735749321.702659]: reset action history +[INFO] [1735749324.411030]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735749327.897215]: resetting environment +[INFO] [1735749327.900470]: cleared current plan +[INFO] [1735749327.901403]: moving to home +Move to target timeout: 5.000817775726318. Trans error: 0.005839774385094643. Rot error: 0.6151033988999629 +Move to target timeout: 5.099645137786865. Trans error: 0.006442955653378188. Rot error: 1.600622754556436 +[INFO] [1735749339.004670]: reached home +[INFO] [1735749339.005930]: reset action history +[INFO] [1735749341.714176]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/episode_rosbags/episode_0_2025-01-01-11-35-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1795426.log +started roslaunch server http://158.130.50.37:43043/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1795692] +process[print_text-2]: started with pid [1795716] +process[rosbag_record-3]: started with pid [1795717] +started bagging! +For topic gripper_width: timestamp difference is -250590153 for nearest: 1735749362378624134 - target: 1735749362629214287 at idx 249 +[INFO] [1735749368.713844]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit[print_text-2] killing on exit +[INFO] [1735749369.450892]: Stored episode 1. +[INFO] [1735749369.451514]: Saving video... +[INFO] [1735749369.652016]: Video saved! +Starting episode 1 +[INFO] [1735749369.670224]: resetting environment +[INFO] [1735749369.680949]: cleared current plan +[INFO] [1735749369.681656]: moving to home +Move to target timeout: 5.001591444015503. Trans error: 0.008680586703121662. Rot error: 1.3335791733129732 +Move to target timeout: 5.099754810333252. Trans error: 0.006291099956706258. Rot error: 1.0385855038717977 +[INFO] [1735749380.785262]: reached home +[INFO] [1735749380.785752]: reset action history +[INFO] [1735749383.501889]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735749386.205490]: resetting environment +[INFO] [1735749386.208586]: cleared current plan +[INFO] [1735749386.208785]: moving to home +Move to target timeout: 5.001219034194946. Trans error: 0.006093483883887529. Rot error: 0.6955657105006933 +Move to target timeout: 5.099586725234985. Trans error: 0.006301306052771093. Rot error: 1.521098447838006 +[INFO] [1735749397.311510]: reached home +[INFO] [1735749397.311916]: reset action history +[INFO] [1735749400.018538]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/episode_rosbags/episode_1_2025-01-01-11-36-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1795426.log +started roslaunch server http://158.130.50.37:39557/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1795828] +process[print_text-5]: started with pid [1795829] +process[rosbag_record-6]: started with pid [1795830] +started bagging! +[INFO] [1735749426.971385]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1735749427.750297]: Stored episode 2. +[INFO] [1735749427.750539]: Saving video... +[INFO] [1735749428.247918]: Video saved! +Starting episode 2 +[INFO] [1735749428.276001]: resetting environment +[INFO] [1735749428.283803]: cleared current plan +[INFO] [1735749428.284524]: moving to home +Move to target timeout: 5.0019824504852295. Trans error: 0.00907379575073719. Rot error: 1.4770952936519968 +Move to target timeout: 5.09878945350647. Trans error: 0.006251439675851949. Rot error: 1.0245763014745646 +[INFO] [1735749439.394477]: reached home +[INFO] [1735749439.405565]: reset action history +[INFO] [1735749442.182854]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735749443.831531]: resetting environment +[INFO] [1735749443.841836]: cleared current plan +[INFO] [1735749443.842057]: moving to home +Move to target timeout: 5.001068830490112. Trans error: 0.006062377244234085. Rot error: 0.6846099392969389 +Move to target timeout: 5.099238872528076. Trans error: 0.006313087596012924. Rot error: 1.525368227992656 +[INFO] [1735749454.950768]: reached home +[INFO] [1735749454.951689]: reset action history +[INFO] [1735749457.661463]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/episode_rosbags/episode_2_2025-01-01-11-37-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1795426.log +started roslaunch server http://158.130.50.37:40139/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1795964] +process[print_text-8]: started with pid [1795965] +process[rosbag_record-9]: started with pid [1795989] +started bagging! +For topic gripper_width: timestamp difference is 130127609 for nearest: 1735749462645311773 - target: 1735749462515184164 at idx 69 +[INFO] [1735749484.608112]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1735749485.405984]: Stored episode 3. +[INFO] [1735749485.408851]: Saving video... +[INFO] [1735749485.939573]: Video saved! +Starting episode 3 +[INFO] [1735749485.982244]: resetting environment +[INFO] [1735749485.992141]: cleared current plan +[INFO] [1735749485.992346]: moving to home +[WARN] [1735749485.941500]: Plan exhausted +[WARN] [1735749485.978552]: Plan exhausted +Move to target timeout: 5.0009942054748535. Trans error: 0.009181719273328781. Rot error: 1.5100310487503135 +Move to target timeout: 5.0962865352630615. Trans error: 0.006242939375799609. Rot error: 1.0231230544506797 +[INFO] [1735749497.097609]: reached home +[INFO] [1735749497.099634]: reset action history +For topic color: timestamp difference is -199843645 for nearest: 1735749499784857988 - target: 1735749499984701633 at idx 29 +[INFO] [1735749500.055903]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735749506.065905]: resetting environment +[INFO] [1735749506.073703]: cleared current plan +[INFO] [1735749506.076472]: moving to home +Move to target timeout: 5.000847578048706. Trans error: 0.005907465238124132. Rot error: 0.6387278449795406 +Move to target timeout: 5.00082540512085. Trans error: 0.0059783064931678755. Rot error: 1.3936883047140105 +[INFO] [1735749517.082841]: reached home +[INFO] [1735749517.084313]: reset action history +[INFO] [1735749519.861065]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/episode_rosbags/episode_3_2025-01-01-11-38-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1795426.log +started roslaunch server http://158.130.50.37:38147/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1796104] +process[print_text-11]: started with pid [1796105] +process[rosbag_record-12]: started with pid [1796129] +started bagging! +[INFO] [1735749546.851363]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735749547.679984]: Stored episode 4. +[INFO] [1735749547.686690]: Saving video... +[INFO] [1735749548.363692]: Video saved! +[WARN] [1735749548.185245]: Plan exhausted +[WARN] [1735749548.222193]: Plan exhausted +[WARN] [1735749548.270588]: Plan exhausted +[WARN] [1735749548.301686]: Plan exhausted +[WARN] [1735749548.344747]: Plan exhausted +[WARN] [1735749548.381823]: Plan exhausted +[WARN] [1735749548.421654]: Plan exhausted +Starting episode 4 +[INFO] [1735749548.409524]: resetting environment +[INFO] [1735749548.441644]: cleared current plan +[INFO] [1735749548.444058]: moving to home +Move to target timeout: 5.000787019729614. Trans error: 0.00809940043836832. Rot error: 1.2510286084291238 +Move to target timeout: 5.097226142883301. Trans error: 0.006247855590675685. Rot error: 1.0318217791190394 +[INFO] [1735749559.558142]: reached home +[INFO] [1735749559.570343]: reset action history +[INFO] [1735749562.360129]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735749573.305260]: resetting environment +[INFO] [1735749573.319735]: cleared current plan +[INFO] [1735749573.323509]: moving to home +Move to target timeout: 5.00252890586853. Trans error: 0.006067171227186918. Rot error: 0.6827877957956108 +Move to target timeout: 5.096555233001709. Trans error: 0.006174354646394644. Rot error: 1.475334321334342 +[INFO] [1735749584.433457]: reached home +[INFO] [1735749584.443017]: reset action history +[INFO] [1735749587.215426]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/episode_rosbags/episode_4_2025-01-01-11-39-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1795426.log +started roslaunch server http://158.130.50.37:39273/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1796263] +process[print_text-14]: started with pid [1796264] +process[rosbag_record-15]: started with pid [1796265] +started bagging! +For topic gripper_width: timestamp difference is -101602632 for nearest: 1735749588178579253 - target: 1735749588280181885 at idx 15 +[INFO] [1735749614.351742]: Storing episode... +[print_text-14] killing on exit +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1735749615.144924]: Stored episode 5. +[INFO] [1735749615.146283]: Saving video... +[INFO] [1735749615.681744]: Video saved! +[INFO] [1735749615.727318]: resetting environment +[INFO] [1735749615.752530]: cleared current plan +[INFO] [1735749615.757606]: moving to home +[WARN] [1735749615.702210]: Plan exhausted +[WARN] [1735749615.741937]: Plan exhausted +Move to target timeout: 5.001079559326172. Trans error: 0.009219175204634666. Rot error: 1.5228510585830617 +Move to target timeout: 5.093205213546753. Trans error: 0.006199683942962905. Rot error: 1.0396715095591378 +[INFO] [1735749626.873089]: reached home +[INFO] [1735749626.881848]: reset action history +[INFO] [1735749629.689388]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/requirements.txt b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-metadata.json b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b081a65c3bcfdb2bb500b2bb3e6da41647c51b53 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T16:34:52.130976", + "startedAt": "2025-01-01T16:34:51.736821", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3748.949, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4494.85, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.808, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 710.8477401733398 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b055b8eeef9b40b05039bcda1f80bbac45877556 --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "e0def80a523fcf0f756011882fd1834f31c8bbbd1dda82834e01408b874db2c5", "size": 1114521, "path": "media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4"}, "global_step": 665, "_timestamp": 1735749634.050535, "_runtime": 342.3054909706116, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "506a301f481b1b43d783f0b1729c200967015613cb8c856d8cf24bf5f28ad82c", "size": 1094264, "path": "media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4"}, "eval/num_success": 0.0, "episode": 50.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "1267f74013e09ddf74b1ce386eab8874b7c3ee631cbe123b20883233c218e800", "size": 1102081, "path": "media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d3882f24ae83d74169feb1e0493e723f6ebaa6fb560c0fec5e726e5e633db37f", "size": 1142341, "path": "media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "50280c95fd108de973aa051d15fbb1a2aa9c77df14a27e53f640fcf41ae576bf", "size": 1134352, "path": "media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://deh3099nfg5jn3ucfsevaqdfz4y2xc4ugg035txmrtdyu8di099ps154l5o0zr4k28huzdq2v2dyctau5gbmmxy0jpqovk61k3sjtfpzz7hap3lxkmpodns0z87bro2s/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://b9j9lm4esnwr2qgxsoolosm37ajf8ez4mnbozq3yl45eudfeciklkr8ssg6fa8b2tqtag8xbpf6nonxdopy9vn3tzocalz57df5an0q7o7kq7voex5my0rmuumimg4n1:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 23, "total_num_episodes": 50, "total_success_rate": 0.46, "total_success_rate_lb_0.01": 0.3002996743124774, "total_success_rate_ub_0.01": 0.6245906352747645, "total_success_rate_lb_0.025": 0.32984319332664913, "total_success_rate_ub_0.025": 0.6058678477790812, "total_success_rate_lb_0.05": 0.34994972500225807, "total_success_rate_ub_0.05": 0.5732010900827871, "total_success_rate_lb_0.1": 0.36585705730021656, "total_success_rate_ub_0.1": 0.5579705722514082, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "e51bb77ae886ca4acc6a04cde1bea310699e2bb433601a0f457a7b09562fd8db", "size": 509, "artifact_path": "wandb-client-artifact://vr7yn8imayxxovrz7a0w7738uo6e0sgbh8rt1kxkjpuq0e2r7wky098x83rd426980dx20tffr7j71idbkd6rrl6smsdtnsfs34mug5j9x939llhx2b8pjwz9pkeje95/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://fsk9s9q1i1xcr5w338bbnrbqcn3u30y7i4u80mhgm9273w8b71kbmts21w5i9hd6tqiqkeow6rb087fx8ozubhbv80z30wr3e5yo3mwnop8gvwhwoaxdc0evkm7pb595:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 342}, "eval/total_success_rate": 0.46000000834465027, "eval/total_success_rate_lb_0.01": 0.30029967427253723, "eval/total_success_rate_ub_0.01": 0.6245906352996826, "eval/total_success_rate_lb_0.025": 0.3298431932926178, "eval/total_success_rate_ub_0.025": 0.605867862701416, "eval/total_success_rate_lb_0.05": 0.3499497175216675, "eval/total_success_rate_ub_0.05": 0.573201060295105, "eval/total_success_rate_lb_0.1": 0.3658570647239685, "eval/total_success_rate_ub_0.1": 0.5579705834388733} \ No newline at end of file diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug-internal.log b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b3f3e35140152fbea2c3fb6663659e1a6cedc56e --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug-internal.log @@ -0,0 +1,892 @@ +2025-01-01 11:34:51,744 INFO StreamThr :1795590 [internal.py:wandb_internal():85] W&B internal server running at pid: 1795590, started at: 2025-01-01 11:34:51.744537 +2025-01-01 11:34:51,746 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:34:51,757 INFO WriterThread:1795590 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/run-4k07t1xl.wandb +2025-01-01 11:34:51,758 DEBUG SenderThread:1795590 [sender.py:send():379] send: header +2025-01-01 11:34:51,764 DEBUG SenderThread:1795590 [sender.py:send():379] send: run +2025-01-01 11:34:52,022 INFO SenderThread:1795590 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files +2025-01-01 11:34:52,022 INFO SenderThread:1795590 [sender.py:_start_run_threads():1188] run started: 4k07t1xl with start time 1735749291.745044 +2025-01-01 11:34:52,038 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:34:52,038 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:34:52,113 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:34:52,118 DEBUG HandlerThread:1795590 [system_info.py:__init__():26] System info init +2025-01-01 11:34:52,118 DEBUG HandlerThread:1795590 [system_info.py:__init__():41] System info init done +2025-01-01 11:34:52,118 INFO HandlerThread:1795590 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:34:52,118 INFO SystemMonitor:1795590 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:34:52,119 INFO HandlerThread:1795590 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:34:52,119 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:34:52,120 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:34:52,120 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:34:52,121 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:34:52,121 INFO SystemMonitor:1795590 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:34:52,130 DEBUG HandlerThread:1795590 [system_info.py:probe():152] Probing system +2025-01-01 11:34:52,132 DEBUG HandlerThread:1795590 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:34:52,140 DEBUG HandlerThread:1795590 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:34:52,140 DEBUG HandlerThread:1795590 [system_info.py:probe():200] Probing system done +2025-01-01 11:34:52,140 DEBUG HandlerThread:1795590 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:34:52.130976', 'startedAt': '2025-01-01T16:34:51.736821', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.949, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4494.85, 'min': 800.0, 'max': 4700.0}, {'current': 4498.808, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 710.8477401733398}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:34:52,140 INFO HandlerThread:1795590 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:34:52,140 INFO HandlerThread:1795590 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:34:52,140 DEBUG HandlerThread:1795590 [system_info.py:_save_code():44] Saving code +2025-01-01 11:34:52,142 DEBUG HandlerThread:1795590 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:34:52,142 DEBUG HandlerThread:1795590 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:34:52,204 DEBUG HandlerThread:1795590 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:34:52,204 INFO HandlerThread:1795590 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:34:52,208 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:34:52,208 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:34:52,208 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:34:52,209 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:34:52,459 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:34:52,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:34:52,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:34:52,461 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:34:52,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:52,462 INFO HandlerThread:1795590 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/tb" + save: true +} +control { + relay_id: "8569b3a4bd464a76b8bdd8443aa0b49c" +} +_info { + stream_id: "4k07t1xl" +} + +2025-01-01 11:34:52,463 DEBUG HandlerThread:1795590 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:34:52,504 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/j7w6apqv-wandb-metadata.json +2025-01-01 11:34:52,516 INFO wandb-upload_2:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/n06ek16b-diff.patch +2025-01-01 11:34:52,519 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/ulu6svnf-code/FISH/eval_robot.py +2025-01-01 11:34:52,601 DEBUG SenderThread:1795590 [sender.py:send():379] send: telemetry +2025-01-01 11:34:52,601 DEBUG SenderThread:1795590 [sender.py:send():379] send: tbrecord +2025-01-01 11:34:52,601 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:34:52,601 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 with policy live +2025-01-01 11:34:52,856 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/q3984jc4-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:34:53,023 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code/FISH/eval_robot.py +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/diff.patch +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/requirements.txt +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-metadata.json +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code +2025-01-01 11:34:53,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code/FISH +2025-01-01 11:34:53,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:54,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:55,023 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:34:55,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:56,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:56,762 DEBUG SenderThread:1795590 [sender.py:send():379] send: config +2025-01-01 11:34:56,762 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:57,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:58,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:59,024 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:34:59,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:00,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:01,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:01,773 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:02,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:03,025 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:03,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:04,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:05,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:06,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:06,873 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:07,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:07,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:07,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:08,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:09,027 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:09,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:10,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:11,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:12,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:12,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:13,028 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:13,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:14,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:15,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:16,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:17,029 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:17,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:17,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:18,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:19,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:20,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:21,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:22,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:22,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:22,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:22,612 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:23,031 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml +2025-01-01 11:35:23,031 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:23,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:24,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:25,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:26,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:27,032 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:27,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:27,897 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:28,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:29,032 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:29,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:30,466 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:31,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:32,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:32,904 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:33,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:34,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:35,034 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:35,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:36,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:37,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:37,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:37,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:38,004 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:38,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:39,035 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:39,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:40,469 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:41,036 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:41,469 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:42,477 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:43,037 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:35:43,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:43,480 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:44,478 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:45,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:46,478 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:47,478 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:48,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:49,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:49,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:50,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:51,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:52,121 DEBUG SystemMonitor:1795590 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:35:52,122 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:35:52,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:35:52,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:35:52,501 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:53,486 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:54,486 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:54,605 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:35:55,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:56,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:57,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:58,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:35:59,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:00,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:00,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:01,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:02,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:03,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:04,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:05,042 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:05,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:05,778 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:06,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:07,460 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:07,460 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:07,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:08,489 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:09,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:09,697 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:36:09,698 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:36:09,698 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 with policy now +2025-01-01 11:36:09,698 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:36:09,698 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:36:09,699 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:36:09,913 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/gai242ym-media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:36:10,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos +2025-01-01 11:36:10,280 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/gdnq6obg-media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 +2025-01-01 11:36:10,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:11,044 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:11,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:11,699 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:12,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:13,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:14,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:15,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:16,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:16,725 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:17,046 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:17,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:18,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:19,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:20,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:21,047 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:21,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:21,829 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:22,122 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:36:22,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:22,464 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:22,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:23,048 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:23,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:24,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:25,049 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:25,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:26,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:27,050 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:27,050 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:36:27,243 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:27,298 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/vj3z41ys-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:36:27,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:28,491 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:29,050 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:29,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:30,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:31,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:32,253 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:32,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:33,052 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:33,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:34,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:35,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:36,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:37,312 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:37,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:37,461 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:37,492 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:38,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:39,054 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:39,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:40,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:41,054 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:41,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:42,471 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:42,493 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:43,055 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:36:43,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:44,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:45,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:46,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:47,471 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:47,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:48,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:49,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:50,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:51,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:52,123 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:36:52,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:36:52,461 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:36:52,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:52,525 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:53,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:54,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:55,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:56,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:57,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:58,472 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:36:58,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:36:59,495 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:00,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:01,497 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:02,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:03,472 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:03,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:04,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:05,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:06,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:07,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:07,462 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:07,496 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:08,268 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:37:08,268 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 with policy now +2025-01-01 11:37:08,271 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:37:08,271 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:37:08,271 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:37:08,273 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:37:08,476 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/p2o572sw-media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 +2025-01-01 11:37:08,497 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:09,063 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:37:09,063 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:09,064 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 +2025-01-01 11:37:09,064 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:37:09,285 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:09,317 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/715folg8-media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 +2025-01-01 11:37:09,497 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:10,499 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:11,064 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:11,500 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:12,500 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:13,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:14,333 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:14,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:15,065 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:15,501 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:16,502 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:17,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:18,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:19,398 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:19,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:20,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:21,067 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:21,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:22,123 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:37:22,461 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:22,462 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:22,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:23,068 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:23,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:24,068 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:37:24,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:24,859 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:24,924 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:37:24,924 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:37:24,925 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:37:25,069 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:37:25,069 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:25,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:26,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:27,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:28,503 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:29,504 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:29,889 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:30,504 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:31,071 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:31,504 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:32,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:33,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:34,505 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:34,951 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:35,072 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:35,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:36,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:37,073 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:37,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:37,462 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:37,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:38,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:39,074 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:39,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:40,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:40,902 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:41,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:42,506 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:43,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:44,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:45,076 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:37:45,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:46,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:46,694 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:47,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:48,507 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:49,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:50,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:51,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:52,124 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:37:52,124 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:52,462 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:37:52,462 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:37:52,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:53,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:54,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:55,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:56,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:57,477 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:37:57,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:58,508 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:37:59,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:00,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:01,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:02,477 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:02,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:03,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:04,509 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:05,512 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:05,981 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:38:05,982 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:38:05,982 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 with policy now +2025-01-01 11:38:05,982 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:38:05,982 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:38:05,983 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:38:06,087 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:38:06,087 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 +2025-01-01 11:38:06,087 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:38:06,186 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/mm7fzv1s-media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 +2025-01-01 11:38:06,429 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/q5yk7gbg-media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 +2025-01-01 11:38:06,515 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:07,087 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:07,472 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:07,472 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:07,514 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:07,555 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:08,088 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:08,515 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:09,517 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:10,516 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:11,517 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:12,518 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:13,089 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:13,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:13,520 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:14,526 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:15,525 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:16,521 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:17,091 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:17,523 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:18,479 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:18,524 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:19,092 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:19,525 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:20,520 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:21,092 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:21,523 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:22,124 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:38:22,467 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:22,468 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:22,521 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:23,524 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:23,620 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:24,527 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:25,530 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:26,094 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:38:26,323 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/6hmvm48u-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:38:26,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:26,777 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:38:26,777 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:38:26,777 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:38:27,094 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:38:27,095 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:27,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:28,526 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:29,480 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:29,528 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:30,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:31,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:32,530 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:33,096 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:33,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:34,481 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:34,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:35,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:36,529 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:37,098 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:37,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:37,464 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:37,530 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:38,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:39,098 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:39,523 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:39,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:40,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:41,099 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:38:41,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:42,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:43,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:44,531 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:45,482 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:45,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:46,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:47,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:48,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:49,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:50,482 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:50,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:51,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:52,124 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:38:52,463 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:38:52,464 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:38:52,532 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:53,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:54,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:55,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:55,616 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:38:56,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:57,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:58,533 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:38:59,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:00,534 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:01,482 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:01,534 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:02,534 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:03,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:04,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:05,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:06,483 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:06,535 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:07,468 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:07,468 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:07,536 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:08,404 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:39:08,404 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 with policy now +2025-01-01 11:39:08,405 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:39:08,407 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:39:08,407 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:39:08,407 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:39:08,543 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:08,641 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/1mg1d2yl-media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 +2025-01-01 11:39:09,108 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:39:09,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:09,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 +2025-01-01 11:39:09,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:39:09,371 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/sv7kqvuk-media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 +2025-01-01 11:39:09,547 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:10,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:10,545 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:11,109 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:11,545 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:12,447 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:12,546 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:13,546 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:14,549 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:15,110 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:15,557 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:16,548 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:17,486 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:17,554 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:18,593 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:19,559 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:20,557 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:21,112 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:21,556 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:22,125 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:39:22,464 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:22,465 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:22,554 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:22,581 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:23,113 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:23,555 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:24,556 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:25,558 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:26,558 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:27,558 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:28,490 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:28,564 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:29,562 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:30,560 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:31,569 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:32,568 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:33,571 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:33,856 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:39:33,856 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:39:33,856 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:33,857 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:39:34,117 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:39:34,117 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:39:34,345 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/77h6cx4p-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:39:34,562 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:35,117 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:35,564 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:36,567 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:37,469 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:37,469 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:37,566 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:38,572 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:39,118 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:39,373 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:39,568 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:40,568 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:41,569 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:42,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:43,571 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:44,437 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:44,573 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:45,120 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:45,575 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:46,572 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:47,121 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:47,593 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:48,575 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:49,122 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:39:49,575 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:50,375 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:50,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:51,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:52,125 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:39:52,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:39:52,465 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:39:52,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:53,576 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:54,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:55,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:55,604 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:39:56,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:57,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:58,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:39:59,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:00,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:01,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:01,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:02,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:03,577 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:04,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:05,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:06,487 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:06,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:07,465 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:40:07,465 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:40:07,578 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:08,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:09,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:10,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:11,555 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:11,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:12,579 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:13,580 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:14,580 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:15,130 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:15,586 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:15,725 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:40:15,725 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:40:15,725 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 with policy now +2025-01-01 11:40:15,726 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:15,726 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:15,726 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:15,930 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/xjtbx3fk-media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 +2025-01-01 11:40:16,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:40:16,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 +2025-01-01 11:40:16,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval +2025-01-01 11:40:16,368 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/pglqyh5i-media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 +2025-01-01 11:40:16,582 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:16,760 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:17,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:17,591 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:18,131 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:18,605 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:19,588 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:20,603 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:21,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:21,809 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:22,126 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:40:22,475 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:40:22,475 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:40:22,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:23,133 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:23,601 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:24,594 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:25,602 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:26,602 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:26,875 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:27,134 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:27,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:28,600 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:29,135 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:29,599 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:30,594 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:31,136 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:31,596 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:32,494 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:32,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:33,137 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:40:33,525 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:40:33,525 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:40:33,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:33,743 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:40:33,936 INFO wandb-upload_0:1795590 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpo6zacjce +2025-01-01 11:40:34,136 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:40:34,137 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:40:34,137 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:40:34,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media +2025-01-01 11:40:34,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval +2025-01-01 11:40:34,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table +2025-01-01 11:40:34,145 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:40:34,292 INFO SenderThread:1795590 [sender.py:send_request_log_artifact():1525] logged artifact run-4k07t1xl-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5NjQ0MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MzU4Nw==', 'latestArtifact': None}} +2025-01-01 11:40:34,292 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:40:34,293 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 11:40:34,293 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:34,293 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,293 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,294 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:34,294 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,294 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,294 DEBUG SenderThread:1795590 [sender.py:send():379] send: telemetry +2025-01-01 11:40:34,294 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,295 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,295 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,295 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,295 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,296 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,296 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,296 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,296 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,296 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,297 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,297 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,297 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,297 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,298 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,298 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,298 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,298 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,298 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,299 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,299 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:34,299 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:34,299 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:40:34,463 INFO wandb-upload_2:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/1ffrey2x-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:40:34,596 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:34,617 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/4xojhgjr-events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:40:34,703 INFO wandb-upload_2:1795590 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpt1oljlbs +2025-01-01 11:40:34,730 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/bh2etqow-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:40:35,097 INFO SenderThread:1795590 [sender.py:send_request_log_artifact():1525] logged artifact run-4k07t1xl-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5NjQ0OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MzU4OA==', 'latestArtifact': None}} +2025-01-01 11:40:35,097 DEBUG SenderThread:1795590 [sender.py:send():379] send: files +2025-01-01 11:40:35,097 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json with policy now +2025-01-01 11:40:35,097 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:35,098 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:35,098 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:35,098 DEBUG SenderThread:1795590 [sender.py:send():379] send: exit +2025-01-01 11:40:35,098 INFO SenderThread:1795590 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:40:35,099 INFO SenderThread:1795590 [sender.py:send_exit():588] handling runtime: 342 +2025-01-01 11:40:35,102 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:35,102 INFO SenderThread:1795590 [sender.py:send_exit():594] send defer +2025-01-01 11:40:35,102 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,102 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:40:35,103 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:35,103 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:40:35,103 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:40:35,103 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,103 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:40:35,103 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:35,103 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:40:35,103 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:40:35,103 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,103 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:40:35,103 INFO HandlerThread:1795590 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:40:35,103 DEBUG SystemMonitor:1795590 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:40:35,103 DEBUG SystemMonitor:1795590 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:40:35,103 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:40:35,104 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:40:35,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:40:35,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:35,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json +2025-01-01 11:40:35,138 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval +2025-01-01 11:40:35,158 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:40:35,159 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:40:35,159 INFO HandlerThread:1795590 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:40:35,159 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:35,159 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:40:35,159 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:40:35,159 DEBUG SenderThread:1795590 [sender.py:send():379] send: stats +2025-01-01 11:40:35,160 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,160 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:40:35,160 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:35,160 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:40:35,160 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:40:35,160 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:35,160 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:40:35,311 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/3vxst9c0-media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json +2025-01-01 11:40:35,556 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /tmp/tmpakapdlk8wandb/2wdrelvx-media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json +2025-01-01 11:40:37,139 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:41,934 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:41,934 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:41,935 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:40:41,935 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:41,936 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:40:41,936 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:40:41,936 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:41,936 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:41,936 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:40:41,936 DEBUG SenderThread:1795590 [sender.py:send():379] send: history +2025-01-01 11:40:41,937 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:40:41,937 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:41,937 DEBUG SenderThread:1795590 [sender.py:send():379] send: summary +2025-01-01 11:40:41,941 INFO SenderThread:1795590 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:40:41,941 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:41,941 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:40:41,942 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:40:41,942 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:41,942 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:40:41,942 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:41,942 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:40:41,945 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:40:41,963 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:42,062 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:40:42,062 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:42,062 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:42,062 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:40:42,062 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:42,062 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:40:42,141 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml +2025-01-01 11:40:42,141 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:40:42,963 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:44,127 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:40:44,128 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:44,128 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:44,128 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:40:44,128 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:44,128 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:40:44,128 INFO SenderThread:1795590 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:40:44,130 INFO SenderThread:1795590 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:40:44,132 INFO SenderThread:1795590 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:40:44,142 INFO Thread-12 :1795590 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:44,145 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:40:44,145 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:44,145 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:40:44,146 DEBUG SenderThread:1795590 [sender.py:send():379] send: artifact +2025-01-01 11:40:44,352 INFO SenderThread:1795590 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:40:44,353 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:44,353 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:40:44,353 INFO SenderThread:1795590 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:40:44,964 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:45,142 INFO SenderThread:1795590 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml config.yaml +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json wandb-summary.json +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 events.out.tfevents.1735749292.leonmkim-ROG-Strix-G15CS-G15CS.1795426.0 +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/requirements.txt requirements.txt +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log output.log +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/diff.patch diff.patch +2025-01-01 11:40:45,143 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:40:45,149 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 media/videos/eval/4_eval_4_50280c95fd108de973aa.mp4 +2025-01-01 11:40:45,149 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 media/videos/eval/3_eval_3_d3882f24ae83d74169fe.mp4 +2025-01-01 11:40:45,149 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 media/videos/eval/1_eval_1_506a301f481b1b43d783.mp4 +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 media/videos/eval/0_eval_0_e0def80a523fcf0f7560.mp4 +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 media/videos/eval/2_eval_2_1267f74013e09ddf74b1.mp4 +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json media/table/eval/total_success_rate_ci_6_e51bb77ae886ca4acc6a.table.json +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:40:45,150 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:40:45,150 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:45,150 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,150 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:40:45,151 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,151 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:40:45,151 INFO SenderThread:1795590 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:40:45,324 INFO wandb-upload_1:1795590 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/wandb-summary.json +2025-01-01 11:40:45,328 INFO wandb-upload_3:1795590 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/config.yaml +2025-01-01 11:40:45,330 INFO wandb-upload_2:1795590 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/requirements.txt +2025-01-01 11:40:45,397 INFO wandb-upload_0:1795590 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/files/output.log +2025-01-01 11:40:45,597 INFO Thread-11 (_thread_body):1795590 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:40:45,597 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,597 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:40:45,597 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,597 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:40:45,597 INFO SenderThread:1795590 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:40:45,599 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:40:45,599 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,599 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:40:45,599 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,599 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:40:45,599 INFO SenderThread:1795590 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:40:45,673 INFO SenderThread:1795590 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:40:45,673 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:40:45,673 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,673 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:40:45,673 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,673 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:40:45,673 INFO SenderThread:1795590 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:40:45,674 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:40:45,674 INFO HandlerThread:1795590 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:40:45,674 DEBUG SenderThread:1795590 [sender.py:send():379] send: final +2025-01-01 11:40:45,674 DEBUG SenderThread:1795590 [sender.py:send():379] send: footer +2025-01-01 11:40:45,674 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: defer +2025-01-01 11:40:45,674 INFO SenderThread:1795590 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:40:45,674 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:45,675 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:45,675 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:40:45,675 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:40:45,675 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:40:45,675 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:40:45,677 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:40:45,678 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:40:45,744 DEBUG SenderThread:1795590 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:40:45,744 INFO MainThread:1795590 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:40:45,745 INFO MainThread:1795590 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:40:45,745 INFO MainThread:1795590 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:40:45,745 DEBUG HandlerThread:1795590 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:40:45,745 INFO HandlerThread:1795590 [handler.py:finish():882] shutting down handler +2025-01-01 11:40:46,676 INFO WriterThread:1795590 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/run-4k07t1xl.wandb +2025-01-01 11:40:46,744 INFO SenderThread:1795590 [sender.py:finish():1615] shutting down sender +2025-01-01 11:40:46,744 INFO SenderThread:1795590 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:40:46,744 INFO SenderThread:1795590 [file_pusher.py:join():175] waiting for file pusher diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug.log b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..46240777867f79a2d3b94a431a589e224f60853e --- /dev/null +++ b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Configure stats pid to 1795426 +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/settings +2025-01-01 11:34:51,738 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug.log +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/logs/debug-internal.log +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl'} +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:init():619] starting backend +2025-01-01 11:34:51,739 INFO MainThread:1795426 [wandb_init.py:init():623] setting up manager +2025-01-01 11:34:51,743 INFO MainThread:1795426 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:34:51,744 INFO MainThread:1795426 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:34:51,757 INFO MainThread:1795426 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:34:51,762 INFO MainThread:1795426 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:34:52,037 INFO MainThread:1795426 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:34:52,106 INFO MainThread:1795426 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:34:52,107 INFO MainThread:1795426 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:34:52,460 INFO MainThread:1795426 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:34:52,460 INFO MainThread:1795426 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:34:52,461 INFO MainThread:1795426 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:34:52,461 INFO MainThread:1795426 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:34:52,462 INFO MainThread:1795426 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:34:52,462 INFO MainThread:1795426 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/4k07t1xl/tb, True +2025-01-01 11:34:56,761 INFO MainThread:1795426 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-01 11:40:46,746 WARNING MsgRouterThr:1795426 [router.py:message_loop():77] message_loop has been closed diff --git a/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/run-4k07t1xl.wandb b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/run-4k07t1xl.wandb new file mode 100644 index 0000000000000000000000000000000000000000..804057aa88e3f7245699404d18264dc5b0d2539e Binary files /dev/null and b/4k07t1xl/wandb/run-20250101_113451-4k07t1xl/run-4k07t1xl.wandb differ diff --git a/7ja33cok/.hydra/config.yaml b/7ja33cok/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/7ja33cok/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/7ja33cok/.hydra/hydra.yaml b/7ja33cok/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a98b356dfae9819cc4ff8bcbd3a53c9d9ca331fb --- /dev/null +++ b/7ja33cok/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7ja33cok + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/7ja33cok/.hydra/overrides.yaml b/7ja33cok/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/7ja33cok/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/7ja33cok/eval_policy.log b/7ja33cok/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..199ccb9e24333b3beac81ac94409c43fc66faba7 --- /dev/null +++ b/7ja33cok/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-01 10:51:36,878][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 10:51:36,881][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-01 10:51:39,042][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/7krw5tkq/episode_rosbags/aligned_depth_to_color_K.npy b/7krw5tkq/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/7krw5tkq/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/7krw5tkq/episode_rosbags/cam_tf_world.npy b/7krw5tkq/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/7krw5tkq/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/7krw5tkq/episode_rosbags/color_K.npy b/7krw5tkq/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/7krw5tkq/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/7krw5tkq/episode_rosbags/depth_K.npy b/7krw5tkq/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/7krw5tkq/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/7krw5tkq/episode_rosbags/episode_4_2025-01-01-11-33-45.bag b/7krw5tkq/episode_rosbags/episode_4_2025-01-01-11-33-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..04258f70ff7274ab6cf91aafffc98133609980cf --- /dev/null +++ b/7krw5tkq/episode_rosbags/episode_4_2025-01-01-11-33-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75ad36631586d82930b0b8faf5174531888303d3647f57462ba8447f5dcaf96c +size 704847865 diff --git a/7krw5tkq/eval_robot.log b/7krw5tkq/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..6a0ebab096abc23844bc30ea2622cc822d72e749 --- /dev/null +++ b/7krw5tkq/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 11:28:37,487][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:28:37,490][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/7krw5tkq/wandb/debug-internal.log b/7krw5tkq/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..734b90923caf07472220a4f13260044ba6ae53bf --- /dev/null +++ b/7krw5tkq/wandb/debug-internal.log @@ -0,0 +1,892 @@ +2025-01-01 11:28:43,224 INFO StreamThr :1794543 [internal.py:wandb_internal():85] W&B internal server running at pid: 1794543, started at: 2025-01-01 11:28:43.224317 +2025-01-01 11:28:43,226 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:28:43,243 INFO WriterThread:1794543 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/run-7krw5tkq.wandb +2025-01-01 11:28:43,243 DEBUG SenderThread:1794543 [sender.py:send():379] send: header +2025-01-01 11:28:43,250 DEBUG SenderThread:1794543 [sender.py:send():379] send: run +2025-01-01 11:28:43,455 INFO SenderThread:1794543 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files +2025-01-01 11:28:43,455 INFO SenderThread:1794543 [sender.py:_start_run_threads():1188] run started: 7krw5tkq with start time 1735748923.224622 +2025-01-01 11:28:43,478 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:28:43,478 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:28:43,553 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:28:43,557 DEBUG HandlerThread:1794543 [system_info.py:__init__():26] System info init +2025-01-01 11:28:43,558 DEBUG HandlerThread:1794543 [system_info.py:__init__():41] System info init done +2025-01-01 11:28:43,558 INFO HandlerThread:1794543 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:28:43,558 INFO SystemMonitor:1794543 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:28:43,558 INFO HandlerThread:1794543 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:28:43,558 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:28:43,559 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:28:43,560 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:28:43,560 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:28:43,561 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:28:43,573 DEBUG HandlerThread:1794543 [system_info.py:probe():152] Probing system +2025-01-01 11:28:43,574 DEBUG HandlerThread:1794543 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:28:43,579 DEBUG HandlerThread:1794543 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:28:43,579 DEBUG HandlerThread:1794543 [system_info.py:probe():200] Probing system done +2025-01-01 11:28:43,579 DEBUG HandlerThread:1794543 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:28:43.573411', 'startedAt': '2025-01-01T16:28:43.213336', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.9015, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.45, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 4500.013, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 704.9242362976074}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:28:43,579 INFO HandlerThread:1794543 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:28:43,579 INFO HandlerThread:1794543 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:28:43,579 DEBUG HandlerThread:1794543 [system_info.py:_save_code():44] Saving code +2025-01-01 11:28:43,581 DEBUG HandlerThread:1794543 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:28:43,581 DEBUG HandlerThread:1794543 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:28:43,645 DEBUG HandlerThread:1794543 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:28:43,645 INFO HandlerThread:1794543 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:28:43,648 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:28:43,648 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:28:43,649 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:28:43,649 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:28:43,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:28:43,889 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:28:43,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:43,889 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:43,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:43,891 INFO HandlerThread:1794543 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/tb" + save: true +} +control { + relay_id: "c285d8b49483481285043490fdcd8b85" +} +_info { + stream_id: "7krw5tkq" +} + +2025-01-01 11:28:43,891 DEBUG HandlerThread:1794543 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:28:43,922 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/vrmuqfut-wandb-metadata.json +2025-01-01 11:28:43,926 INFO wandb-upload_1:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/fg6whylx-code/FISH/eval_robot.py +2025-01-01 11:28:43,947 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/ree151az-diff.patch +2025-01-01 11:28:43,977 DEBUG SenderThread:1794543 [sender.py:send():379] send: telemetry +2025-01-01 11:28:43,977 DEBUG SenderThread:1794543 [sender.py:send():379] send: tbrecord +2025-01-01 11:28:43,978 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:28:43,978 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 with policy live +2025-01-01 11:28:44,210 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/7m5mz9cw-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/diff.patch +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/requirements.txt +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/code/FISH/eval_robot.py +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-metadata.json +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:28:44,458 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/code/FISH +2025-01-01 11:28:44,458 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/code +2025-01-01 11:28:44,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:45,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:46,456 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:28:46,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:47,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:48,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:48,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:49,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:50,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:51,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:52,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:53,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:53,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:54,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:55,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:56,190 DEBUG SenderThread:1794543 [sender.py:send():379] send: config +2025-01-01 11:28:56,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:57,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:58,459 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:28:58,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:58,889 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:58,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:58,964 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:59,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:00,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:01,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:02,461 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:02,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:03,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:04,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:04,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:05,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:06,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:07,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:08,462 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:08,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:09,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:10,013 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:10,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:11,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:12,463 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:12,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:13,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:13,889 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:13,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:14,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:15,025 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:15,464 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/config.yaml +2025-01-01 11:29:15,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:16,465 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:16,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:17,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:18,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:19,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:20,165 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:20,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:21,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:22,466 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:22,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:23,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:24,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:25,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:25,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:26,467 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:26,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:27,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:28,468 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:28,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:28,890 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:28,937 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:29,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:30,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:30,991 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:31,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:32,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:33,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:34,469 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:34,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:35,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:36,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:36,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:37,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:38,470 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:38,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:39,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:40,471 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:40,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:41,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:42,466 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:42,472 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:42,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:43,561 DEBUG SystemMonitor:1794543 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:29:43,562 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:29:43,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:43,890 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:43,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:44,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:45,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:46,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:47,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:48,002 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:48,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:49,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:50,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:51,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:52,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:53,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:53,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:54,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:55,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:56,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:57,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:58,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:58,890 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:58,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:59,015 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:59,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:00,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:01,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:02,901 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:03,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:04,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:04,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:05,902 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:06,902 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:07,902 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:08,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:09,132 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:30:09,132 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 with policy now +2025-01-01 11:30:09,132 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:30:09,133 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:30:09,133 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:30:09,135 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:30:09,404 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/ydfr40ny-media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:30:09,747 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/mtqwumgx-media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 +2025-01-01 11:30:09,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:10,146 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:10,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:10,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:11,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:12,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:13,562 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:30:13,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:13,891 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:13,937 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:14,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:15,193 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:15,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:16,481 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:16,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:17,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:18,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:19,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:20,249 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:20,482 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:20,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:21,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:22,483 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:22,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:23,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:24,484 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:24,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:25,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:25,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:26,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:27,485 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:30:27,735 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/zren7kmq-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:30:27,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:28,485 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:28,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:28,891 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:28,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:29,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:30,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:31,051 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:31,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:32,487 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:32,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:33,907 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:34,907 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:35,907 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:36,667 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:36,907 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:37,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:38,489 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:38,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:39,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:40,489 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:40,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:41,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:42,490 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:42,626 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:42,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:43,563 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:30:43,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:43,891 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:43,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:44,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:45,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:46,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:47,911 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:47,967 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:48,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:49,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:50,493 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:50,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:51,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:52,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:53,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:53,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:54,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:55,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:56,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:57,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:58,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:58,891 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:58,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:59,053 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:59,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:00,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:01,911 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:02,911 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:03,911 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:04,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:04,916 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:05,917 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:06,916 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:07,917 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:08,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:08,766 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:31:08,766 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 with policy now +2025-01-01 11:31:08,767 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:31:08,767 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:31:08,768 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:31:08,774 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:31:08,917 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:09,026 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/rs6n7a19-media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 +2025-01-01 11:31:09,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:31:09,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 +2025-01-01 11:31:09,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:31:09,785 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/yvpw51hj-media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 +2025-01-01 11:31:09,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:09,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:10,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:10,919 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:11,501 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:11,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:12,919 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:13,563 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:31:13,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:13,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:13,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:14,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:14,969 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:15,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:16,502 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:16,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:17,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:18,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:19,941 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:20,504 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:20,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:20,941 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:21,926 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:22,504 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:22,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:23,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:24,505 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:24,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:25,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:26,828 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:26,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:27,362 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:31:27,363 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:31:27,363 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:31:27,506 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:31:27,506 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:31:27,756 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/d9ukhhbb-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:31:27,922 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:28,507 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:28,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:28,892 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:28,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:29,922 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:30,922 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:31,852 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:31,923 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:32,927 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:33,923 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:34,509 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:34,926 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:35,924 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:36,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:36,924 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:37,924 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:38,510 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:38,924 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:39,925 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:40,511 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:40,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:41,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:41,981 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:42,511 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:42,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:43,564 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:31:43,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:43,909 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:43,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:44,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:45,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:46,513 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:46,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:47,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:47,981 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:48,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:49,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:50,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:51,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:52,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:53,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:53,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:54,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:55,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:56,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:57,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:58,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:58,893 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:58,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:59,000 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:59,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:00,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:01,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:02,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:03,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:04,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:04,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:05,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:06,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:07,938 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:08,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:08,935 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:08,955 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:32:08,955 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 with policy now +2025-01-01 11:32:09,001 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:32:09,002 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:32:09,002 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:32:09,005 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:32:09,289 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/hby0yuh8-media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 +2025-01-01 11:32:09,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:32:09,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 +2025-01-01 11:32:09,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:32:09,901 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/rwrh5u2l-media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 +2025-01-01 11:32:09,935 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:10,005 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:10,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:10,940 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:11,940 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:12,940 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:13,564 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:32:13,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:13,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:13,943 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:14,943 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:15,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:15,941 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:16,523 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:16,945 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:17,943 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:18,946 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:19,949 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:20,099 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:20,525 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:20,948 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:21,948 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:22,525 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:22,945 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:23,951 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:24,526 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:24,951 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:25,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:25,949 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:26,953 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:27,955 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:28,901 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:28,901 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:28,951 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:29,954 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:30,953 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:31,222 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:32:31,222 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:32:31,222 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:31,223 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:32:31,528 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:32:31,529 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:32:31,779 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/4rpyncsp-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:32:31,954 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:32,529 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:32,955 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:33,952 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:34,953 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:35,952 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:36,530 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:36,665 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:36,956 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:37,954 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:38,957 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:39,957 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:40,958 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:41,728 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:41,963 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:42,532 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:42,965 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:43,564 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:32:43,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:43,893 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:43,966 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:44,533 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:44,958 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:45,958 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:46,534 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:46,958 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:47,712 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:47,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:48,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:49,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:50,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:51,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:52,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:52,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:53,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:54,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:55,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:56,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:57,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:58,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:58,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:58,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:58,992 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:59,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:00,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:01,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:02,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:03,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:04,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:04,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:05,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:06,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:07,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:08,541 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:08,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:09,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:09,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:10,962 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:11,965 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:12,542 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:12,674 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:33:12,674 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 with policy now +2025-01-01 11:33:12,679 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:33:12,679 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:33:12,679 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:33:12,680 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:33:12,959 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/rbd0i1kg-media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 +2025-01-01 11:33:12,962 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:13,543 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:33:13,543 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 +2025-01-01 11:33:13,543 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:33:13,565 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:33:13,894 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/vv5utxx8-media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 +2025-01-01 11:33:13,902 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:13,902 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:13,966 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:14,543 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:14,973 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:15,025 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:15,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:16,973 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:17,977 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:18,544 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:18,977 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:19,970 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:20,914 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:20,980 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:21,972 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:22,974 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:23,976 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:24,546 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:24,974 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:25,975 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:26,547 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:26,801 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:26,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:27,977 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:28,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:28,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:28,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:29,978 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:30,978 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:31,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:32,295 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:33:32,295 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:33:32,295 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:32,296 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:33:32,549 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:32,549 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:33:32,549 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:33:32,820 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/v207z1qo-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:33:32,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:33,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:34,982 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:35,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:36,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:37,829 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:37,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:38,551 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:38,983 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:39,986 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:40,991 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:41,994 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:42,888 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:42,990 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:43,565 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:33:43,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:43,896 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:43,986 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:44,553 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:44,989 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:45,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:46,554 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:46,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:47,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:48,555 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:48,669 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:48,985 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:49,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:50,555 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:50,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:51,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:52,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:53,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:53,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:54,557 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:54,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:55,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:56,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:57,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:58,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:58,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:58,988 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:59,010 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:59,990 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:00,559 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:00,588 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:34:00,588 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 with policy now +2025-01-01 11:34:00,590 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:34:00,590 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:00,590 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:00,591 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:00,847 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/g48dsy20-media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 +2025-01-01 11:34:00,989 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:01,559 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:01,559 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 +2025-01-01 11:34:01,560 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:34:01,868 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/6e60bsk6-media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 +2025-01-01 11:34:01,993 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:02,560 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:02,995 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:03,560 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:03,994 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:04,915 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:05,003 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:06,001 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:06,561 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:07,008 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:08,000 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:09,009 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:09,922 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:10,003 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:11,004 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:12,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:12,563 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:13,007 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:13,566 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:34:13,901 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:34:13,901 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:34:14,005 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:15,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:15,713 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:16,015 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:16,564 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:17,017 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:18,018 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:19,015 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:19,350 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:19,350 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:19,350 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:19,565 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:19,566 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:34:20,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:20,048 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:34:20,049 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:34:20,212 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:34:20,566 INFO wandb-upload_2:1794543 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6bulox_k +2025-01-01 11:34:20,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:20,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:34:20,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:34:20,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media +2025-01-01 11:34:20,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table +2025-01-01 11:34:20,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval +2025-01-01 11:34:20,589 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:34:20,597 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:34:20,984 INFO SenderThread:1794543 [sender.py:send_request_log_artifact():1525] logged artifact run-7krw5tkq-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5MjI2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MjkzNg==', 'latestArtifact': None}} +2025-01-01 11:34:20,984 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:34:20,984 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 11:34:20,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:20,985 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:20,985 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:20,993 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:20,994 DEBUG SenderThread:1794543 [sender.py:send():379] send: telemetry +2025-01-01 11:34:20,994 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,008 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,008 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,009 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,009 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:21,025 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,025 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,026 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,026 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,047 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,047 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,048 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,048 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,069 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,070 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,070 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,070 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,088 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,089 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,107 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,107 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,124 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,124 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:34:21,165 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/ktqrg3f1-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:34:21,407 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/r2jegu31-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:34:21,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:21,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json +2025-01-01 11:34:21,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval +2025-01-01 11:34:21,604 INFO wandb-upload_0:1794543 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnj7wzogk +2025-01-01 11:34:21,752 INFO wandb-upload_1:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/h1z6bubv-media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json +2025-01-01 11:34:22,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:22,019 INFO SenderThread:1794543 [sender.py:send_request_log_artifact():1525] logged artifact run-7krw5tkq-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5MjI3NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MjkzOQ==', 'latestArtifact': None}} +2025-01-01 11:34:22,020 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:34:22,020 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json with policy now +2025-01-01 11:34:22,020 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:22,020 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:22,024 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:22,256 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/35ok2z6u-media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json +2025-01-01 11:34:22,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:22,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:23,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:23,334 DEBUG SenderThread:1794543 [sender.py:send():379] send: exit +2025-01-01 11:34:23,334 INFO SenderThread:1794543 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:34:23,334 INFO SenderThread:1794543 [sender.py:send_exit():588] handling runtime: 339 +2025-01-01 11:34:23,334 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:23,334 INFO SenderThread:1794543 [sender.py:send_exit():594] send defer +2025-01-01 11:34:23,334 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,334 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:34:23,335 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:23,335 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:34:23,335 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:34:23,335 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,335 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:34:23,335 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:23,335 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:34:23,335 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:34:23,335 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,335 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:34:23,335 INFO HandlerThread:1794543 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:34:23,335 DEBUG SystemMonitor:1794543 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:34:23,335 DEBUG SystemMonitor:1794543 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:34:23,336 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:34:23,336 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:34:23,387 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:34:23,388 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:34:23,388 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:34:23,388 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:23,388 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:34:23,388 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:34:23,388 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:34:23,389 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,389 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:34:23,389 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:23,389 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:34:23,389 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:34:23,389 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,389 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:34:23,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:24,571 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:30,356 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:30,356 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:30,356 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:34:30,357 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:30,357 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:34:30,357 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:34:30,357 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:30,358 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:30,358 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:34:30,358 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:30,358 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:30,359 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:30,359 DEBUG SenderThread:1794543 [sender.py:send():379] send: summary +2025-01-01 11:34:30,363 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:30,363 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:30,363 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:34:30,363 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:34:30,363 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:30,363 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:34:30,363 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:30,363 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:34:30,367 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:30,471 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:34:30,471 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:30,471 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:34:30,471 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:30,471 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:34:30,573 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/config.yaml +2025-01-01 11:34:30,573 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:31,335 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:32,893 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:34:32,893 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:32,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:32,894 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:34:32,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:32,894 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:34:32,894 INFO SenderThread:1794543 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:34:32,896 INFO SenderThread:1794543 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:34:32,898 INFO SenderThread:1794543 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:34:32,908 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:34:32,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:32,909 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:34:32,909 DEBUG SenderThread:1794543 [sender.py:send():379] send: artifact +2025-01-01 11:34:33,115 INFO SenderThread:1794543 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:34:33,115 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:33,115 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:34:33,115 INFO SenderThread:1794543 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:34:33,336 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:33,574 INFO SenderThread:1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:33,575 INFO SenderThread:1794543 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files +2025-01-01 11:34:33,575 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/config.yaml config.yaml +2025-01-01 11:34:33,575 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json wandb-summary.json +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/requirements.txt requirements.txt +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log output.log +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/diff.patch diff.patch +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 +2025-01-01 11:34:33,578 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 +2025-01-01 11:34:33,579 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 +2025-01-01 11:34:33,579 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 +2025-01-01 11:34:33,580 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 +2025-01-01 11:34:33,583 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json +2025-01-01 11:34:33,584 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:34:33,584 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:34:33,584 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:33,584 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:33,584 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:34:33,584 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:33,584 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:34:33,584 INFO SenderThread:1794543 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:34:33,762 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:33,775 INFO wandb-upload_1:1794543 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/config.yaml +2025-01-01 11:34:33,798 INFO wandb-upload_4:1794543 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:33,831 INFO wandb-upload_3:1794543 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/requirements.txt +2025-01-01 11:34:33,855 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/satjes7p-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:34:34,056 INFO Thread-11 (_thread_body):1794543 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:34:34,056 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:34,056 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:34:34,056 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:34,056 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:34:34,056 INFO SenderThread:1794543 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:34:34,057 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:34:34,058 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:34,058 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:34:34,058 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:34,058 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:34:34,058 INFO SenderThread:1794543 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:34:34,123 INFO SenderThread:1794543 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:34:34,123 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:34:34,123 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:34,124 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:34:34,124 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:34,124 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:34:34,124 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:34:34,124 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:34,124 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:34:34,124 DEBUG SenderThread:1794543 [sender.py:send():379] send: final +2025-01-01 11:34:34,124 DEBUG SenderThread:1794543 [sender.py:send():379] send: footer +2025-01-01 11:34:34,124 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:34,125 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:34:34,125 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:34,125 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:34:34,126 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:34,126 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:34:34,127 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:34:34,127 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:34,127 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:34:34,127 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:34,195 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:34,195 INFO MainThread:1794543 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:34:34,196 INFO MainThread:1794543 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:34:34,196 INFO MainThread:1794543 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:34:34,196 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:34:34,196 INFO HandlerThread:1794543 [handler.py:finish():882] shutting down handler +2025-01-01 11:34:35,127 INFO WriterThread:1794543 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/run-7krw5tkq.wandb +2025-01-01 11:34:35,195 INFO SenderThread:1794543 [sender.py:finish():1615] shutting down sender +2025-01-01 11:34:35,195 INFO SenderThread:1794543 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:34:35,196 INFO SenderThread:1794543 [file_pusher.py:join():175] waiting for file pusher diff --git a/7krw5tkq/wandb/debug.log b/7krw5tkq/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7c8203683eb8ad350f75880fabdcefd0c67a1a9b --- /dev/null +++ b/7krw5tkq/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:28:43,218 INFO MainThread:1794357 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:28:43,218 INFO MainThread:1794357 [wandb_setup.py:_flush():76] Configure stats pid to 1794357 +2025-01-01 11:28:43,218 INFO MainThread:1794357 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:28:43,218 INFO MainThread:1794357 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/settings +2025-01-01 11:28:43,218 INFO MainThread:1794357 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:28:43,218 INFO MainThread:1794357 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:28:43,218 INFO MainThread:1794357 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:28:43,218 INFO MainThread:1794357 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:28:43,219 INFO MainThread:1794357 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/logs/debug.log +2025-01-01 11:28:43,219 INFO MainThread:1794357 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/logs/debug-internal.log +2025-01-01 11:28:43,219 INFO MainThread:1794357 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:28:43,219 INFO MainThread:1794357 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq'} +2025-01-01 11:28:43,219 INFO MainThread:1794357 [wandb_init.py:init():619] starting backend +2025-01-01 11:28:43,219 INFO MainThread:1794357 [wandb_init.py:init():623] setting up manager +2025-01-01 11:28:43,223 INFO MainThread:1794357 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:28:43,224 INFO MainThread:1794357 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:28:43,242 INFO MainThread:1794357 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:28:43,248 INFO MainThread:1794357 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:28:43,477 INFO MainThread:1794357 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:28:43,547 INFO MainThread:1794357 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:28:43,548 INFO MainThread:1794357 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:28:43,889 INFO MainThread:1794357 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:28:43,889 INFO MainThread:1794357 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:28:43,889 INFO MainThread:1794357 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:28:43,889 INFO MainThread:1794357 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:28:43,891 INFO MainThread:1794357 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:28:43,891 INFO MainThread:1794357 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/tb, True +2025-01-01 11:28:56,190 INFO MainThread:1794357 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-01 11:34:35,197 WARNING MsgRouterThr:1794357 [router.py:message_loop():77] message_loop has been closed diff --git a/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/run-7krw5tkq.wandb b/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/run-7krw5tkq.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3723397ef42f7a412614d704ad1091ee35749324 Binary files /dev/null and b/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/run-7krw5tkq.wandb differ diff --git a/b2veaffg/.hydra/config.yaml b/b2veaffg/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/b2veaffg/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/b2veaffg/.hydra/hydra.yaml b/b2veaffg/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1adf9791af825d212f34308737ea159382f5a1d3 --- /dev/null +++ b/b2veaffg/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/b2veaffg + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/b2veaffg/.hydra/overrides.yaml b/b2veaffg/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/b2veaffg/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/b2veaffg/eval_policy.log b/b2veaffg/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..a67ef2f546586ba335caae6c0db84204151c10ec --- /dev/null +++ b/b2veaffg/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-01 11:28:35,503][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:28:35,506][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-01 11:28:38,814][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d7689cb06a28d6108fc7712c4e8e265b3bc94f1 --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1007_0' +wandb_notes: 1007_0_restarted_2 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/ex6qcxor/.hydra/config.yaml b/ex6qcxor/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/ex6qcxor/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ex6qcxor/.hydra/hydra.yaml b/ex6qcxor/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cd3828740c29641dd53e3030e08b374de3d5957 --- /dev/null +++ b/ex6qcxor/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/ex6qcxor + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ex6qcxor/.hydra/overrides.yaml b/ex6qcxor/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ex6qcxor/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ex6qcxor/eval_policy.log b/ex6qcxor/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..4086a82c4ff1ac15e6d50c39a2577d389a04a71a --- /dev/null +++ b/ex6qcxor/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-31 21:03:02,603][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 21:03:02,606][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-31 21:03:06,550][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/hab6e0c2/.hydra/config.yaml b/hab6e0c2/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/hab6e0c2/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/hab6e0c2/.hydra/hydra.yaml b/hab6e0c2/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..579d96a6607c852a2c36624c7853d137f9a502cc --- /dev/null +++ b/hab6e0c2/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/hab6e0c2/.hydra/overrides.yaml b/hab6e0c2/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/hab6e0c2/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/hab6e0c2/episode_rosbags/aligned_depth_to_color_K.npy b/hab6e0c2/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hab6e0c2/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hab6e0c2/episode_rosbags/cam_tf_world.npy b/hab6e0c2/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/hab6e0c2/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/hab6e0c2/episode_rosbags/color_K.npy b/hab6e0c2/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hab6e0c2/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hab6e0c2/episode_rosbags/depth_K.npy b/hab6e0c2/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/hab6e0c2/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/hab6e0c2/eval_robot.log b/hab6e0c2/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..748187cdc1803c519413218390264ea842849001 --- /dev/null +++ b/hab6e0c2/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 22:41:18,939][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 22:41:18,942][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/hab6e0c2/eval_video/0_eval.mp4 b/hab6e0c2/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b64f1de55aa59b2b4fe36e053a831bd0f0f3f07 --- /dev/null +++ b/hab6e0c2/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a609f5ccf30787a62b54dec4e5e3d7b7a5c630c65ce17333b6113b9b91190bf +size 1096264 diff --git a/hab6e0c2/eval_video/1_eval.mp4 b/hab6e0c2/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b82ab2bd9a8f224ac3d231498b4e256bb32f1bcb --- /dev/null +++ b/hab6e0c2/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18c2b938cd7782cd9f25ec9f00ee72c14c279ca3126cd25e302ca8c08cf84953 +size 1094935 diff --git a/hab6e0c2/eval_video/2_eval.mp4 b/hab6e0c2/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a8602f804dd1e1f3c736d671f5fd925c01d2d7a --- /dev/null +++ b/hab6e0c2/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:994bf0b5346b4ff89086b91dce6cb717be086123e0a6fde3e895113c730e6a8f +size 1095592 diff --git a/hab6e0c2/eval_video/3_eval.mp4 b/hab6e0c2/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a7f16eb14e9fcc90a2668496538a1d6ebf35aa54 --- /dev/null +++ b/hab6e0c2/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e78024cdb724dea405d62f5e91a9212e4062445b813f813203076f713f681328 +size 1116866 diff --git a/hab6e0c2/eval_video/4_eval.mp4 b/hab6e0c2/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b034a6d2bd27f140a424a98ba19324352bc8e9af --- /dev/null +++ b/hab6e0c2/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2efe40fe909e07fe4bc555a259eedecfd56bc22579410f8f15f447125bfa08da +size 1102653 diff --git a/hab6e0c2/tb/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 b/hab6e0c2/tb/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 new file mode 100644 index 0000000000000000000000000000000000000000..aa58935c45e44ea28dda07d89bb115f3a873ca38 --- /dev/null +++ b/hab6e0c2/tb/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:949cba2a393d494dfc96ca5960d25e1448cd957e9760b3e3f402200cfd3dfd23 +size 6539 diff --git a/hab6e0c2/wandb/debug-internal.log b/hab6e0c2/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a5d929d65786fa6356d8d77aeac9f3c7becea2d9 --- /dev/null +++ b/hab6e0c2/wandb/debug-internal.log @@ -0,0 +1,771 @@ +2024-12-31 22:41:24,651 INFO StreamThr :1761088 [internal.py:wandb_internal():85] W&B internal server running at pid: 1761088, started at: 2024-12-31 22:41:24.651388 +2024-12-31 22:41:24,652 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status +2024-12-31 22:41:24,664 INFO WriterThread:1761088 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/run-hab6e0c2.wandb +2024-12-31 22:41:24,665 DEBUG SenderThread:1761088 [sender.py:send():379] send: header +2024-12-31 22:41:24,672 DEBUG SenderThread:1761088 [sender.py:send():379] send: run +2024-12-31 22:41:24,875 INFO SenderThread:1761088 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files +2024-12-31 22:41:24,875 INFO SenderThread:1761088 [sender.py:_start_run_threads():1188] run started: hab6e0c2 with start time 1735702884.652187 +2024-12-31 22:41:24,892 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 22:41:24,892 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: check_version +2024-12-31 22:41:24,937 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 22:41:24,942 DEBUG HandlerThread:1761088 [system_info.py:__init__():26] System info init +2024-12-31 22:41:24,942 DEBUG HandlerThread:1761088 [system_info.py:__init__():41] System info init done +2024-12-31 22:41:24,942 INFO HandlerThread:1761088 [system_monitor.py:start():194] Starting system monitor +2024-12-31 22:41:24,942 INFO SystemMonitor:1761088 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 22:41:24,943 INFO HandlerThread:1761088 [system_monitor.py:probe():214] Collecting system info +2024-12-31 22:41:24,943 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 22:41:24,943 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started disk monitoring +2024-12-31 22:41:24,944 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 22:41:24,945 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started memory monitoring +2024-12-31 22:41:24,946 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started network monitoring +2024-12-31 22:41:24,956 DEBUG HandlerThread:1761088 [system_info.py:probe():152] Probing system +2024-12-31 22:41:24,957 DEBUG HandlerThread:1761088 [system_info.py:_probe_git():137] Probing git +2024-12-31 22:41:24,962 DEBUG HandlerThread:1761088 [system_info.py:_probe_git():145] Probing git done +2024-12-31 22:41:24,962 DEBUG HandlerThread:1761088 [system_info.py:probe():200] Probing system done +2024-12-31 22:41:24,962 DEBUG HandlerThread:1761088 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:41:24.956057', 'startedAt': '2025-01-01T03:41:24.641097', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.99775, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.994, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 572.5552635192871}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 22:41:24,962 INFO HandlerThread:1761088 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 22:41:24,962 INFO HandlerThread:1761088 [system_monitor.py:probe():227] Publishing system info +2024-12-31 22:41:24,962 DEBUG HandlerThread:1761088 [system_info.py:_save_code():44] Saving code +2024-12-31 22:41:24,964 DEBUG HandlerThread:1761088 [system_info.py:_save_code():67] Saving code done +2024-12-31 22:41:24,964 DEBUG HandlerThread:1761088 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 22:41:25,025 DEBUG HandlerThread:1761088 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 22:41:25,025 INFO HandlerThread:1761088 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 22:41:25,028 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:41:25,028 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 22:41:25,029 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 22:41:25,029 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 22:41:25,268 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 22:41:25,268 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:41:25,269 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: python_packages +2024-12-31 22:41:25,269 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:41:25,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:25,271 INFO HandlerThread:1761088 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/tb" + save: true +} +control { + relay_id: "de7f07f25eba493894dbee4bf8d8f2f2" +} +_info { + stream_id: "hab6e0c2" +} + +2024-12-31 22:41:25,271 DEBUG HandlerThread:1761088 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 22:41:25,286 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/1j0q8qdu-wandb-metadata.json +2024-12-31 22:41:25,313 INFO wandb-upload_2:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/ogtlnwwr-diff.patch +2024-12-31 22:41:25,323 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/kqxzdy5o-code/FISH/eval_robot.py +2024-12-31 22:41:25,386 DEBUG SenderThread:1761088 [sender.py:send():379] send: telemetry +2024-12-31 22:41:25,386 DEBUG SenderThread:1761088 [sender.py:send():379] send: tbrecord +2024-12-31 22:41:25,387 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:41:25,387 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 with policy live +2024-12-31 22:41:25,615 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/aqsy6z34-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:41:25,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:41:25,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-metadata.json +2024-12-31 22:41:25,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code/FISH/eval_robot.py +2024-12-31 22:41:25,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/diff.patch +2024-12-31 22:41:25,878 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/requirements.txt +2024-12-31 22:41:25,878 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:25,878 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code/FISH +2024-12-31 22:41:25,878 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code +2024-12-31 22:41:26,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:27,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:27,876 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:28,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:29,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:29,285 DEBUG SenderThread:1761088 [sender.py:send():379] send: config +2024-12-31 22:41:30,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:30,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:31,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:31,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:32,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:33,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:34,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:35,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:35,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:35,879 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:36,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:37,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:37,879 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:38,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:39,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:40,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:41:40,269 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:41:40,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:40,388 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:41,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:41,880 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:42,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:43,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:44,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:45,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:45,544 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:45,881 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:46,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:47,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:47,883 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:48,275 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:49,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:50,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:51,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:51,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:52,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:53,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:54,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:55,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:41:55,269 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:41:55,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:56,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:56,419 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:56,885 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml +2024-12-31 22:41:57,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:58,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:59,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:00,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:01,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:02,251 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:02,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:03,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:03,887 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:04,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:05,275 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:05,888 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:06,276 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:07,276 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:07,888 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:08,097 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:08,277 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:09,277 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:10,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:42:10,270 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:42:10,278 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:10,966 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:42:10,966 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 with policy now +2024-12-31 22:42:10,969 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:42:10,970 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:42:10,970 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:42:10,971 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:42:11,223 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/2wrh7q1j-media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 +2024-12-31 22:42:11,277 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:11,889 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media +2024-12-31 22:42:12,174 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/ofw7se4b-media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 +2024-12-31 22:42:12,277 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:13,278 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:13,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:13,986 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:14,278 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:15,278 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:15,891 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:16,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:17,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:18,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:19,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:19,446 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:19,892 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:20,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:21,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:22,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:23,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:24,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:24,946 DEBUG SystemMonitor:1761088 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 22:42:24,947 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:42:24,947 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:25,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:42:25,270 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:42:25,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:26,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:27,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:28,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:29,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:30,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:30,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:31,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:32,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:33,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:34,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:35,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:35,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:36,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:37,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:38,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:39,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:39,898 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:42:40,181 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/oypqig34-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:42:40,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:42:40,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:42:40,275 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:40,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:41,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:41,899 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:42,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:43,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:43,900 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:44,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:45,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:45,900 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:46,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:46,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:47,283 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:48,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:49,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:50,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:51,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:51,285 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:52,285 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:53,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:54,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:54,948 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:42:55,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:42:55,272 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:42:55,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:56,285 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:56,398 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:57,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:58,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:59,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:00,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:01,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:01,526 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:02,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:03,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:03,906 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:04,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:05,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:06,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:07,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:07,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:08,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:09,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:10,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:43:10,272 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:43:10,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:11,195 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:43:11,196 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 with policy now +2024-12-31 22:43:11,196 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:43:11,197 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:43:11,197 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:43:11,198 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:43:11,289 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:11,465 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/uizi10i0-media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 +2024-12-31 22:43:11,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:43:11,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:11,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 +2024-12-31 22:43:11,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:43:12,207 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/jjztpo5r-media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 +2024-12-31 22:43:12,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:13,233 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:13,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:13,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:14,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:15,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:15,910 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:16,291 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:17,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:18,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:18,773 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:19,291 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:19,911 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:20,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:21,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:22,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:23,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:24,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:24,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:24,948 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:43:25,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:43:25,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:43:25,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:26,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:26,914 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:43:27,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:27,701 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:43:27,701 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:43:27,701 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:43:27,914 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:43:27,914 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:28,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:29,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:29,702 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:29,915 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:30,325 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:31,294 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:31,916 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:32,294 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:33,294 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:34,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:35,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:35,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:36,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:37,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:38,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:39,296 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:40,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:43:40,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:43:40,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:40,396 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:41,296 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:42,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:43,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:44,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:45,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:46,276 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:46,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:47,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:48,298 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:49,298 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:50,299 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:51,276 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:51,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:52,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:53,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:54,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:54,949 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:43:55,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:43:55,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:43:55,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:56,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:56,380 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:57,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:58,311 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:58,517 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:43:58,517 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 with policy now +2024-12-31 22:43:58,521 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:43:58,522 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:43:58,522 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:43:58,522 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:43:58,772 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/hohl1wu3-media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 +2024-12-31 22:43:58,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:43:58,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 +2024-12-31 22:43:58,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:43:59,220 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/dkcwyxbn-media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 +2024-12-31 22:43:59,306 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:59,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:00,307 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:00,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:01,314 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:02,167 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:02,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:03,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:03,926 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:04,319 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:05,308 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:05,927 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:06,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:07,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:07,314 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:08,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:09,311 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:10,275 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:44:10,275 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:44:10,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:11,314 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:12,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:12,554 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:44:12,554 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:44:12,554 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:12,554 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:44:12,929 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:44:12,929 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:44:13,195 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/vuqz7cpd-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:44:13,314 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:13,930 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:14,315 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:15,324 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:15,930 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:16,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:17,315 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:17,636 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:17,931 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:18,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:19,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:20,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:21,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:21,932 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:22,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:23,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:23,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:24,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:24,949 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:44:25,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:44:25,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:44:25,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:26,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:27,319 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:28,319 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:28,342 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:29,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:30,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:31,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:32,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:33,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:34,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:34,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:35,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:36,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:37,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:38,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:39,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:39,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:40,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:44:40,274 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:44:40,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:41,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:42,322 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:43,322 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:44,327 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:44,717 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:44,892 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:44:44,893 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 with policy now +2024-12-31 22:44:44,901 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:44:44,902 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:44:44,902 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:44:44,902 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:44:44,940 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:44:44,940 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 +2024-12-31 22:44:44,941 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:44:45,157 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/cna1il3e-media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 +2024-12-31 22:44:45,322 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:45,455 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/jsmpnc8i-media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 +2024-12-31 22:44:45,941 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:46,326 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:46,941 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:47,326 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:48,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:49,332 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:49,942 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:50,291 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:50,348 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:51,341 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:52,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:53,333 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:53,943 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:54,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:54,950 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:44:55,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:44:55,279 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:44:55,341 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:55,440 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:56,331 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:56,608 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:44:56,608 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:44:56,610 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:44:56,944 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:44:56,944 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:44:57,333 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:57,945 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:58,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:59,338 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:59,945 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:00,339 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:00,449 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:01,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:01,946 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:02,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:03,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:04,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:05,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:06,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:06,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:07,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:08,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:09,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:10,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:45:10,274 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:45:10,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:11,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:11,382 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:12,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:13,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:14,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:15,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:16,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:17,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:17,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:18,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:19,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:20,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:21,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:22,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:22,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:23,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:24,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:24,950 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:45:25,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:45:25,274 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:45:25,337 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:26,337 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:27,337 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:27,371 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:28,341 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:28,649 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:45:28,649 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 with policy now +2024-12-31 22:45:28,651 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:45:28,652 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:45:28,652 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:28,652 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:28,905 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/ss3ni1qi-media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 +2024-12-31 22:45:28,958 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:45:28,959 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 +2024-12-31 22:45:28,959 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:45:29,230 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/bxmyi9i0-media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 +2024-12-31 22:45:29,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:29,959 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:30,348 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:30,959 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:31,350 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:32,354 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:32,376 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:33,355 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:33,960 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:34,350 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:35,358 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:36,354 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:37,357 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:37,962 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:38,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:38,356 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:39,360 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:40,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:45:40,281 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:45:40,354 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:40,583 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 22:45:40,583 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 22:45:40,792 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table +2024-12-31 22:45:41,037 INFO wandb-upload_1:1761088 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp1zpzrit +2024-12-31 22:45:41,187 INFO wandb-upload_2:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/p2vik5j3-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 22:45:41,195 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 22:45:41,204 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:45:41,225 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/73j5lwt0-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:45:41,350 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:41,382 INFO SenderThread:1761088 [sender.py:send_request_log_artifact():1525] logged artifact run-hab6e0c2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE3MjA5MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3NDY5NA==', 'latestArtifact': None}} +2024-12-31 22:45:41,382 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:45:41,382 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 22:45:41,383 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:45:41,383 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,383 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,383 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:45:41,383 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,384 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,384 DEBUG SenderThread:1761088 [sender.py:send():379] send: telemetry +2024-12-31 22:45:41,384 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,385 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,385 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,385 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,387 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,387 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,388 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,388 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,388 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,388 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,388 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,389 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,389 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,389 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,389 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,390 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,390 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,390 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,390 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,390 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,391 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,391 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,391 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 22:45:41,628 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/qvglvppt-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 22:45:41,728 INFO wandb-upload_0:1761088 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpseruw75k +2024-12-31 22:45:41,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:45:41,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:41,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:45:41,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json +2024-12-31 22:45:41,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval +2024-12-31 22:45:42,142 INFO SenderThread:1761088 [sender.py:send_request_log_artifact():1525] logged artifact run-hab6e0c2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE3MjA5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3NDY5Ng==', 'latestArtifact': None}} +2024-12-31 22:45:42,142 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:45:42,142 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json with policy now +2024-12-31 22:45:42,142 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:45:42,142 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:42,143 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:42,324 INFO wandb-upload_2:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/tupuxcci-media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json +2024-12-31 22:45:42,350 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:42,579 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/zewlda6l-media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json +2024-12-31 22:45:42,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:45:43,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:43,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:44,146 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:44,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:44,965 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml +2024-12-31 22:45:45,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:46,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:47,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:48,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:49,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:49,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:50,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:51,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:52,352 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:53,352 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:54,285 DEBUG SenderThread:1761088 [sender.py:send():379] send: exit +2024-12-31 22:45:54,286 INFO SenderThread:1761088 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 22:45:54,286 INFO SenderThread:1761088 [sender.py:send_exit():588] handling runtime: 269 +2024-12-31 22:45:54,286 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:54,286 INFO SenderThread:1761088 [sender.py:send_exit():594] send defer +2024-12-31 22:45:54,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,286 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 22:45:54,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:54,286 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:45:54,287 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 22:45:54,287 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 1 +2024-12-31 22:45:54,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,287 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 22:45:54,287 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:45:54,287 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 22:45:54,287 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 2 +2024-12-31 22:45:54,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,287 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 22:45:54,287 INFO HandlerThread:1761088 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 22:45:54,287 DEBUG SystemMonitor:1761088 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 22:45:54,287 DEBUG SystemMonitor:1761088 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 22:45:54,288 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 22:45:54,288 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 22:45:54,339 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 22:45:54,339 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 22:45:54,339 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined network monitor +2024-12-31 22:45:54,340 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:45:54,340 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 22:45:54,340 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 3 +2024-12-31 22:45:54,340 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:45:54,340 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,340 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 22:45:54,340 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:45:54,340 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 22:45:54,340 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 4 +2024-12-31 22:45:54,340 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,341 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 22:45:54,969 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:45:55,969 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:46:00,669 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:00,669 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:46:00,669 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 22:46:00,671 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:00,671 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 22:46:00,671 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 5 +2024-12-31 22:46:00,671 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:00,671 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:46:00,671 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:46:00,671 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:00,671 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 22:46:00,672 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:46:00,672 DEBUG SenderThread:1761088 [sender.py:send():379] send: summary +2024-12-31 22:46:00,672 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:46:00,673 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 6 +2024-12-31 22:46:00,673 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:00,673 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 22:46:00,673 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 7 +2024-12-31 22:46:00,673 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:46:00,673 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:00,673 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 22:46:00,673 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 22:46:00,971 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:46:01,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:02,523 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 8 +2024-12-31 22:46:02,523 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:02,524 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:02,524 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 22:46:02,524 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:02,524 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 22:46:02,524 INFO SenderThread:1761088 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 22:46:02,526 INFO SenderThread:1761088 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 22:46:02,528 INFO SenderThread:1761088 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 22:46:02,538 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 9 +2024-12-31 22:46:02,539 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:02,539 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 22:46:02,539 DEBUG SenderThread:1761088 [sender.py:send():379] send: artifact +2024-12-31 22:46:02,725 INFO SenderThread:1761088 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTY3NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzY3NQ==', 'versionIndex': 398}}} +2024-12-31 22:46:02,726 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:02,726 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 22:46:02,726 INFO SenderThread:1761088 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 22:46:02,972 INFO SenderThread:1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml config.yaml +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json wandb-summary.json +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/requirements.txt requirements.txt +2024-12-31 22:46:02,976 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-metadata.json wandb-metadata.json +2024-12-31 22:46:02,976 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log output.log +2024-12-31 22:46:02,976 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/diff.patch diff.patch +2024-12-31 22:46:02,977 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 10 +2024-12-31 22:46:02,982 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:02,982 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 22:46:02,982 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 22:46:03,170 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:46:03,172 INFO wandb-upload_2:1761088 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml +2024-12-31 22:46:03,192 INFO wandb-upload_4:1761088 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:46:03,222 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/fg1qo4kf-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:46:03,241 INFO wandb-upload_3:1761088 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/requirements.txt +2024-12-31 22:46:03,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:03,288 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:03,441 INFO Thread-11 (_thread_body):1761088 [sender.py:transition_state():614] send defer: 11 +2024-12-31 22:46:03,441 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:03,441 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 22:46:03,442 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:03,442 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 22:46:03,442 INFO SenderThread:1761088 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 22:46:03,443 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 12 +2024-12-31 22:46:03,443 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:03,443 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 22:46:03,443 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:03,443 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 22:46:03,443 INFO SenderThread:1761088 [file_stream.py:finish():601] file stream finish called +2024-12-31 22:46:03,618 INFO SenderThread:1761088 [file_stream.py:finish():605] file stream finish is done +2024-12-31 22:46:03,618 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 13 +2024-12-31 22:46:03,618 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:03,618 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 22:46:03,618 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:03,618 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 22:46:03,618 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 14 +2024-12-31 22:46:03,618 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:03,618 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 22:46:03,619 DEBUG SenderThread:1761088 [sender.py:send():379] send: final +2024-12-31 22:46:03,619 DEBUG SenderThread:1761088 [sender.py:send():379] send: footer +2024-12-31 22:46:03,619 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:03,619 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 22:46:03,619 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:03,619 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 22:46:03,620 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:03,620 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:03,620 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: server_info +2024-12-31 22:46:03,620 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 22:46:03,622 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 22:46:03,623 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:46:03,683 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:03,683 INFO MainThread:1761088 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 22:46:03,683 INFO MainThread:1761088 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 22:46:03,684 INFO MainThread:1761088 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 22:46:03,684 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 22:46:03,684 INFO HandlerThread:1761088 [handler.py:finish():882] shutting down handler +2024-12-31 22:46:04,621 INFO WriterThread:1761088 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/run-hab6e0c2.wandb +2024-12-31 22:46:04,683 INFO SenderThread:1761088 [sender.py:finish():1615] shutting down sender +2024-12-31 22:46:04,683 INFO SenderThread:1761088 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 22:46:04,683 INFO SenderThread:1761088 [file_pusher.py:join():175] waiting for file pusher diff --git a/hab6e0c2/wandb/debug.log b/hab6e0c2/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f38d3004666972414e1ba60808835a50ad4c7884 --- /dev/null +++ b/hab6e0c2/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Configure stats pid to 1760903 +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/settings +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug.log +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug-internal.log +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:init():569] calling init triggers +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2'} +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:init():619] starting backend +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:init():623] setting up manager +2024-12-31 22:41:24,650 INFO MainThread:1760903 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 22:41:24,651 INFO MainThread:1760903 [wandb_init.py:init():631] backend started and connected +2024-12-31 22:41:24,664 INFO MainThread:1760903 [wandb_init.py:init():720] updated telemetry +2024-12-31 22:41:24,670 INFO MainThread:1760903 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 22:41:24,891 INFO MainThread:1760903 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 22:41:24,932 INFO MainThread:1760903 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 22:41:24,932 INFO MainThread:1760903 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 22:41:25,268 INFO MainThread:1760903 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 22:41:25,269 INFO MainThread:1760903 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 22:41:25,269 INFO MainThread:1760903 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 22:41:25,269 INFO MainThread:1760903 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 22:41:25,270 INFO MainThread:1760903 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 22:41:25,270 INFO MainThread:1760903 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/tb, True +2024-12-31 22:41:29,284 INFO MainThread:1760903 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2024-12-31 22:46:04,685 WARNING MsgRouterThr:1760903 [router.py:message_loop():77] message_loop has been closed diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code/FISH/eval_robot.py b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..6f971c04f88c3f951a2321c29af50f00efe5b3a1 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + total_num_episodes += filtered_run.summary_metrics['episode'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6d3b1deca5fe540eafe7e9364c9da40c3875cba --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1007_0' +wandb_notes: + desc: null + value: 1007_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735702884 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/diff.patch b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..df1b3966b7d19592540b568ba2e88fffb49e0995 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/diff.patch @@ -0,0 +1,68 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..b9e3dfe 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history ++# wandb_run_id: '1002_0' ++ ++# all books, 6/20 demos per book ++# RGBD+mask+act history + wandb_run_id: '1007_0' + ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..6f971c0 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json new file mode 100644 index 0000000000000000000000000000000000000000..266c22571991d798a58a7aea9027d97be212a363 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[14, 50, 0.28, 0.14942845709182756, 0.4434052407854736, 0.17244662351006784, 0.42392848433585506, 0.18811421847620696, 0.3895118534785946, 0.19997028639952735, 0.37491621824620447]]} \ No newline at end of file diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b64f1de55aa59b2b4fe36e053a831bd0f0f3f07 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a609f5ccf30787a62b54dec4e5e3d7b7a5c630c65ce17333b6113b9b91190bf +size 1096264 diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b82ab2bd9a8f224ac3d231498b4e256bb32f1bcb --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18c2b938cd7782cd9f25ec9f00ee72c14c279ca3126cd25e302ca8c08cf84953 +size 1094935 diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a8602f804dd1e1f3c736d671f5fd925c01d2d7a --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:994bf0b5346b4ff89086b91dce6cb717be086123e0a6fde3e895113c730e6a8f +size 1095592 diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a7f16eb14e9fcc90a2668496538a1d6ebf35aa54 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e78024cdb724dea405d62f5e91a9212e4062445b813f813203076f713f681328 +size 1116866 diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b034a6d2bd27f140a424a98ba19324352bc8e9af --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2efe40fe909e07fe4bc555a259eedecfd56bc22579410f8f15f447125bfa08da +size 1102653 diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..43ab23973f47a167d137930db5f74b665e1b7be7 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log @@ -0,0 +1,284 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735702889.285517]: resetting environment +[INFO] [1735702889.288820]: cleared current plan +[INFO] [1735702889.289910]: moving to home +[INFO] [1735702890.292199]: reached home +[INFO] [1735702890.293400]: reset action history +[INFO] [1735702893.003613]: environment reset +Starting episode 0 +[INFO] [1735702893.005552]: resetting environment +[INFO] [1735702893.009520]: cleared current plan +[INFO] [1735702893.010532]: moving to home +[INFO] [1735702894.012893]: reached home +[INFO] [1735702894.014075]: reset action history +[INFO] [1735702896.722854]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735702899.301121]: resetting environment +[INFO] [1735702899.307281]: cleared current plan +[INFO] [1735702899.308452]: moving to home +[INFO] [1735702900.311112]: reached home +[INFO] [1735702900.312353]: reset action history +[INFO] [1735702903.020731]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/episode_rosbags/episode_0_2024-12-31-22-41-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1760903.log +started roslaunch server http://158.130.50.37:41001/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1761192] +process[print_text-2]: started with pid [1761216] +process[rosbag_record-3]: started with pid [1761217] +started bagging! +For topic gripper_width: timestamp difference is -102233820 for nearest: 1735702905353371210 - target: 1735702905455605030 at idx 35 +For topic gripper_width: timestamp difference is -118763233 for nearest: 1735702922053394531 - target: 1735702922172157764 at idx 249 +For topic gripper_width: timestamp difference is -119783493 for nearest: 1735702924653291134 - target: 1735702924773074627 at idx 249 +For topic gripper_width: timestamp difference is -117085534 for nearest: 1735702925853316945 - target: 1735702925970402479 at idx 249 +[INFO] [1735702930.011824]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735702930.758156]: Stored episode 1. +[INFO] [1735702930.758486]: Saving video... +[INFO] [1735702930.945323]: Video saved! +Starting episode 1 +[INFO] [1735702930.973955]: resetting environment +[INFO] [1735702930.985059]: cleared current plan +[INFO] [1735702930.985307]: moving to home +[INFO] [1735702934.687621]: reached home +[INFO] [1735702934.688110]: reset action history +[INFO] [1735702937.397204]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735702959.223679]: resetting environment +[INFO] [1735702959.228201]: cleared current plan +[INFO] [1735702959.228356]: moving to home +[INFO] [1735702960.229256]: reached home +[INFO] [1735702960.229653]: reset action history +[INFO] [1735702962.938567]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/episode_rosbags/episode_1_2024-12-31-22-42-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1760903.log +started roslaunch server http://158.130.50.37:41811/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1761339] +process[print_text-5]: started with pid [1761340] +process[rosbag_record-6]: started with pid [1761341] +started bagging! +For topic gripper_width: timestamp difference is 103367321 for nearest: 1735702981519985907 - target: 1735702981416618586 at idx 230 +For topic gripper_width: timestamp difference is 103484523 for nearest: 1735702981719939840 - target: 1735702981616455317 at idx 232 +[INFO] [1735702989.906725]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735702990.651865]: Stored episode 2. +[INFO] [1735702990.652162]: Saving video... +[INFO] [1735702991.168148]: Video saved! +Starting episode 2 +[INFO] [1735702991.199359]: resetting environment +[INFO] [1735702991.231241]: cleared current plan +[INFO] [1735702991.231887]: moving to home +[INFO] [1735702994.935911]: reached home +[INFO] [1735702994.940128]: reset action history +[INFO] [1735702997.729175]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735703006.548090]: resetting environment +[INFO] [1735703006.559596]: cleared current plan +[INFO] [1735703006.560329]: moving to home +[INFO] [1735703007.564679]: reached home +[INFO] [1735703007.565248]: reset action history +[INFO] [1735703010.276595]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/episode_rosbags/episode_2_2024-12-31-22-43-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1760903.log +started roslaunch server http://158.130.50.37:43043/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1761473] +process[print_text-8]: started with pid [1761474] +process[rosbag_record-9]: started with pid [1761498] +started bagging! +[INFO] [1735703037.254146]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735703038.038272]: Stored episode 3. +[INFO] [1735703038.039579]: Saving video... +[INFO] [1735703038.481215]: Video saved! +Starting episode 3 +[INFO] [1735703038.525397]: resetting environment +[INFO] [1735703038.547302]: cleared current plan +[INFO] [1735703038.551995]: moving to home +[WARN] [1735703038.408722]: Plan exhausted +[WARN] [1735703038.464326]: Plan exhausted +[WARN] [1735703038.490341]: Plan exhausted +[WARN] [1735703038.529243]: Plan exhausted +[INFO] [1735703042.165054]: reached home +[INFO] [1735703042.170633]: reset action history +[INFO] [1735703044.952512]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735703052.195583]: resetting environment +[INFO] [1735703052.214707]: cleared current plan +[INFO] [1735703052.217507]: moving to home +[INFO] [1735703053.220637]: reached home +[INFO] [1735703053.222588]: reset action history +For topic color: timestamp difference is -133084297 for nearest: 1735703055920549154 - target: 1735703056053633451 at idx 29 +[INFO] [1735703056.137879]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/episode_rosbags/episode_3_2024-12-31-22-44-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1760903.log +started roslaunch server http://158.130.50.37:43151/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1761634] +process[print_text-11]: started with pid [1761635] +process[rosbag_record-12]: started with pid [1761636] +started bagging! +For topic gripper_width: timestamp difference is 104869629 for nearest: 1735703059153316046 - target: 1735703059048446417 at idx 42 +[INFO] [1735703083.364842]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1735703084.134616]: Stored episode 4. +[INFO] [1735703084.140355]: Saving video... +[INFO] [1735703084.851589]: Video saved! +Starting episode 4 +[INFO] [1735703084.906890]: resetting environment +[INFO] [1735703084.929882]: cleared current plan +[INFO] [1735703084.930582]: moving to home +[WARN] [1735703084.713334]: Plan exhausted +[WARN] [1735703084.751425]: Plan exhausted +[WARN] [1735703084.791075]: Plan exhausted +[WARN] [1735703084.830774]: Plan exhausted +[WARN] [1735703084.871921]: Plan exhausted +[WARN] [1735703084.912147]: Plan exhausted +[INFO] [1735703088.533864]: reached home +[INFO] [1735703088.542552]: reset action history +[INFO] [1735703091.348912]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735703096.470960]: resetting environment +[INFO] [1735703096.483771]: cleared current plan +[INFO] [1735703096.483978]: moving to home +[INFO] [1735703097.487267]: reached home +[INFO] [1735703097.490824]: reset action history +[INFO] [1735703100.222391]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/episode_rosbags/episode_4_2024-12-31-22-45-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1760903.log +started roslaunch server http://158.130.50.37:37779/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1761779] +process[print_text-14]: started with pid [1761780] +process[rosbag_record-15]: started with pid [1761781] +started bagging! +[INFO] [1735703127.375652]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1735703128.153291]: Stored episode 5. +[INFO] [1735703128.157740]: Saving video... +[INFO] [1735703128.602480]: Video saved! +[INFO] [1735703128.651079]: resetting environment +[INFO] [1735703128.670363]: cleared current plan +[INFO] [1735703128.670526]: moving to home +[WARN] [1735703128.513855]: Plan exhausted +[WARN] [1735703128.549941]: Plan exhausted +[WARN] [1735703128.590461]: Plan exhausted +[WARN] [1735703128.630354]: Plan exhausted +[WARN] [1735703128.669796]: Plan exhausted +[INFO] [1735703132.375803]: reached home +[INFO] [1735703132.382220]: reset action history +[INFO] [1735703135.204624]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/requirements.txt b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-metadata.json b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7b5393fdd0f927d41285452406596ce821185034 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T03:41:24.956057", + "startedAt": "2025-01-01T03:41:24.641097", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.99775, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.994, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 572.5552635192871 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..da5460f11eaea8d8d02eab68d6e1dcc881ca5f20 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "0a609f5ccf30787a62b54dec4e5e3d7b7a5c630c65ce17333b6113b9b91190bf", "size": 1096264, "path": "media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4"}, "global_step": 662, "_timestamp": 1735703141.0900836, "_runtime": 256.43789649009705, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "18c2b938cd7782cd9f25ec9f00ee72c14c279ca3126cd25e302ca8c08cf84953", "size": 1094935, "path": "media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4"}, "eval/num_success": 0.0, "episode": 50.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "994bf0b5346b4ff89086b91dce6cb717be086123e0a6fde3e895113c730e6a8f", "size": 1095592, "path": "media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "e78024cdb724dea405d62f5e91a9212e4062445b813f813203076f713f681328", "size": 1116866, "path": "media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "2efe40fe909e07fe4bc555a259eedecfd56bc22579410f8f15f447125bfa08da", "size": 1102653, "path": "media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://adjhkta3pz4f94k0dh1ywnno6p8btgak9thornsc7mnoe6frdl1feahwfwon8t5qsflg5z2nac1bmxls2msit80dp8xm4mtdyotuxml7xomh7u8adlwrefk2fwa40zp5/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ry8ip2xuwkn2lpzayvc8tfpv9ext11zw0spmoql4xnqxuchmv6w2atax3na1qpnre4evj6ne04sdx1z6g8xkcljvmool30t6230nwlo5r3mlhh5k4ohwypkb077b6yue:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 14, "total_num_episodes": 50, "total_success_rate": 0.28, "total_success_rate_lb_0.01": 0.14942845709182756, "total_success_rate_ub_0.01": 0.4434052407854736, "total_success_rate_lb_0.025": 0.17244662351006784, "total_success_rate_ub_0.025": 0.42392848433585506, "total_success_rate_lb_0.05": 0.18811421847620696, "total_success_rate_ub_0.05": 0.3895118534785946, "total_success_rate_lb_0.1": 0.19997028639952735, "total_success_rate_ub_0.1": 0.37491621824620447, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "aa5e6a1bd4b6c595ca4af2858c6fe247a974b31d911e16da91ee5aebbbb627a9", "size": 512, "artifact_path": "wandb-client-artifact://wad1ybkhdruz5mjoo2bc3nl4wr9sacki5oey2x53w4msdhj5y2m09qwgsngyowp7xqiiuw3ni59iau1u28t086amrct6xyjvfiw3ym4xkjdyfwk4vdj4rsb955mxzhet/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://lyl76xom7h51anojkshq2nl0l5lpsk800d2bvi537s0k19h051ith4ttq2hon0f3esb8rk2m7tppljbw4zjm6nj2z8ea586j7b0m5ctws5pzxnxlbv0d98woq622tb2z:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 269}, "eval/total_success_rate": 0.2800000011920929, "eval/total_success_rate_lb_0.01": 0.14942845702171326, "eval/total_success_rate_ub_0.01": 0.44340524077415466, "eval/total_success_rate_lb_0.025": 0.1724466234445572, "eval/total_success_rate_ub_0.025": 0.42392849922180176, "eval/total_success_rate_lb_0.05": 0.1881142258644104, "eval/total_success_rate_ub_0.05": 0.3895118534564972, "eval/total_success_rate_lb_0.1": 0.1999702900648117, "eval/total_success_rate_ub_0.1": 0.3749162256717682} \ No newline at end of file diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug-internal.log b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a5d929d65786fa6356d8d77aeac9f3c7becea2d9 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug-internal.log @@ -0,0 +1,771 @@ +2024-12-31 22:41:24,651 INFO StreamThr :1761088 [internal.py:wandb_internal():85] W&B internal server running at pid: 1761088, started at: 2024-12-31 22:41:24.651388 +2024-12-31 22:41:24,652 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status +2024-12-31 22:41:24,664 INFO WriterThread:1761088 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/run-hab6e0c2.wandb +2024-12-31 22:41:24,665 DEBUG SenderThread:1761088 [sender.py:send():379] send: header +2024-12-31 22:41:24,672 DEBUG SenderThread:1761088 [sender.py:send():379] send: run +2024-12-31 22:41:24,875 INFO SenderThread:1761088 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files +2024-12-31 22:41:24,875 INFO SenderThread:1761088 [sender.py:_start_run_threads():1188] run started: hab6e0c2 with start time 1735702884.652187 +2024-12-31 22:41:24,892 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 22:41:24,892 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: check_version +2024-12-31 22:41:24,937 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 22:41:24,942 DEBUG HandlerThread:1761088 [system_info.py:__init__():26] System info init +2024-12-31 22:41:24,942 DEBUG HandlerThread:1761088 [system_info.py:__init__():41] System info init done +2024-12-31 22:41:24,942 INFO HandlerThread:1761088 [system_monitor.py:start():194] Starting system monitor +2024-12-31 22:41:24,942 INFO SystemMonitor:1761088 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 22:41:24,943 INFO HandlerThread:1761088 [system_monitor.py:probe():214] Collecting system info +2024-12-31 22:41:24,943 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 22:41:24,943 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started disk monitoring +2024-12-31 22:41:24,944 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 22:41:24,945 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started memory monitoring +2024-12-31 22:41:24,946 INFO SystemMonitor:1761088 [interfaces.py:start():188] Started network monitoring +2024-12-31 22:41:24,956 DEBUG HandlerThread:1761088 [system_info.py:probe():152] Probing system +2024-12-31 22:41:24,957 DEBUG HandlerThread:1761088 [system_info.py:_probe_git():137] Probing git +2024-12-31 22:41:24,962 DEBUG HandlerThread:1761088 [system_info.py:_probe_git():145] Probing git done +2024-12-31 22:41:24,962 DEBUG HandlerThread:1761088 [system_info.py:probe():200] Probing system done +2024-12-31 22:41:24,962 DEBUG HandlerThread:1761088 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:41:24.956057', 'startedAt': '2025-01-01T03:41:24.641097', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.99775, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.994, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 572.5552635192871}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 22:41:24,962 INFO HandlerThread:1761088 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 22:41:24,962 INFO HandlerThread:1761088 [system_monitor.py:probe():227] Publishing system info +2024-12-31 22:41:24,962 DEBUG HandlerThread:1761088 [system_info.py:_save_code():44] Saving code +2024-12-31 22:41:24,964 DEBUG HandlerThread:1761088 [system_info.py:_save_code():67] Saving code done +2024-12-31 22:41:24,964 DEBUG HandlerThread:1761088 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 22:41:25,025 DEBUG HandlerThread:1761088 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 22:41:25,025 INFO HandlerThread:1761088 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 22:41:25,028 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:41:25,028 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 22:41:25,029 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 22:41:25,029 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 22:41:25,268 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 22:41:25,268 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:41:25,269 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: python_packages +2024-12-31 22:41:25,269 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:41:25,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:25,271 INFO HandlerThread:1761088 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/tb" + save: true +} +control { + relay_id: "de7f07f25eba493894dbee4bf8d8f2f2" +} +_info { + stream_id: "hab6e0c2" +} + +2024-12-31 22:41:25,271 DEBUG HandlerThread:1761088 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 22:41:25,286 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/1j0q8qdu-wandb-metadata.json +2024-12-31 22:41:25,313 INFO wandb-upload_2:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/ogtlnwwr-diff.patch +2024-12-31 22:41:25,323 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/kqxzdy5o-code/FISH/eval_robot.py +2024-12-31 22:41:25,386 DEBUG SenderThread:1761088 [sender.py:send():379] send: telemetry +2024-12-31 22:41:25,386 DEBUG SenderThread:1761088 [sender.py:send():379] send: tbrecord +2024-12-31 22:41:25,387 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:41:25,387 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 with policy live +2024-12-31 22:41:25,615 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/aqsy6z34-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:41:25,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:41:25,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-metadata.json +2024-12-31 22:41:25,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code/FISH/eval_robot.py +2024-12-31 22:41:25,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/diff.patch +2024-12-31 22:41:25,878 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/requirements.txt +2024-12-31 22:41:25,878 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:25,878 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code/FISH +2024-12-31 22:41:25,878 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code +2024-12-31 22:41:26,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:27,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:27,876 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:28,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:29,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:29,285 DEBUG SenderThread:1761088 [sender.py:send():379] send: config +2024-12-31 22:41:30,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:30,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:31,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:31,877 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:32,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:33,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:34,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:35,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:35,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:35,879 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:36,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:37,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:37,879 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:38,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:39,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:40,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:41:40,269 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:41:40,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:40,388 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:41,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:41,880 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:42,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:43,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:44,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:45,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:45,544 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:45,881 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:46,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:47,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:47,883 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:41:48,275 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:49,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:50,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:51,270 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:51,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:52,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:53,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:54,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:55,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:41:55,269 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:41:55,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:56,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:56,419 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:41:56,885 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml +2024-12-31 22:41:57,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:58,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:41:59,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:00,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:01,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:02,251 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:02,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:03,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:03,887 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:04,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:05,275 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:05,888 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:06,276 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:07,276 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:07,888 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:08,097 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:08,277 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:09,277 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:10,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:42:10,270 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:42:10,278 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:10,966 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:42:10,966 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 with policy now +2024-12-31 22:42:10,969 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:42:10,970 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:42:10,970 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:42:10,971 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:42:11,223 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/2wrh7q1j-media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 +2024-12-31 22:42:11,277 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:11,889 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:42:11,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media +2024-12-31 22:42:12,174 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/ofw7se4b-media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 +2024-12-31 22:42:12,277 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:13,278 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:13,890 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:13,986 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:14,278 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:15,278 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:15,891 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:16,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:17,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:18,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:19,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:19,446 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:19,892 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:20,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:21,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:22,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:23,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:24,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:24,946 DEBUG SystemMonitor:1761088 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 22:42:24,947 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:42:24,947 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:25,269 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:42:25,270 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:42:25,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:26,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:27,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:28,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:29,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:30,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:30,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:31,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:32,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:33,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:34,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:35,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:35,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:36,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:37,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:38,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:39,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:39,898 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:42:40,181 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/oypqig34-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:42:40,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:42:40,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:42:40,275 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:40,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:41,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:41,899 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:42,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:43,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:43,900 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:44,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:45,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:45,900 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:42:46,271 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:46,282 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:47,283 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:48,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:49,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:50,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:51,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:51,285 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:52,285 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:53,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:54,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:54,948 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:42:55,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:42:55,272 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:42:55,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:56,285 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:56,398 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:42:57,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:58,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:42:59,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:00,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:01,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:01,526 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:02,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:03,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:03,906 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:04,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:05,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:06,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:07,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:07,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:08,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:09,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:10,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:43:10,272 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:43:10,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:11,195 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:43:11,196 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 with policy now +2024-12-31 22:43:11,196 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:43:11,197 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:43:11,197 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:43:11,198 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:43:11,289 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:11,465 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/uizi10i0-media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 +2024-12-31 22:43:11,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:43:11,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:11,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 +2024-12-31 22:43:11,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:43:12,207 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/jjztpo5r-media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 +2024-12-31 22:43:12,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:13,233 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:13,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:13,909 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:14,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:15,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:15,910 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:16,291 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:17,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:18,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:18,773 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:19,291 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:19,911 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:20,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:21,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:22,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:23,292 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:24,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:24,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:24,948 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:43:25,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:43:25,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:43:25,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:26,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:26,914 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:43:27,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:27,701 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:43:27,701 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:43:27,701 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:43:27,914 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:43:27,914 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:28,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:29,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:29,702 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:29,915 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:30,325 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:31,294 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:31,916 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:43:32,294 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:33,294 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:34,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:35,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:35,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:36,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:37,295 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:38,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:39,296 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:40,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:43:40,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:43:40,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:40,396 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:41,296 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:42,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:43,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:44,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:45,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:46,276 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:46,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:47,297 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:48,298 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:49,298 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:50,299 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:51,276 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:51,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:52,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:53,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:54,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:54,949 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:43:55,272 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:43:55,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:43:55,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:56,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:56,380 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:43:57,300 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:58,311 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:58,517 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:43:58,517 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 with policy now +2024-12-31 22:43:58,521 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:43:58,522 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:43:58,522 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:43:58,522 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:43:58,772 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/hohl1wu3-media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 +2024-12-31 22:43:58,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:43:58,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 +2024-12-31 22:43:58,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:43:59,220 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/dkcwyxbn-media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 +2024-12-31 22:43:59,306 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:43:59,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:00,307 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:00,925 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:01,314 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:02,167 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:02,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:03,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:03,926 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:04,319 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:05,308 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:05,927 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:06,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:07,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:07,314 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:08,313 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:09,311 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:10,275 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:44:10,275 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:44:10,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:11,314 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:12,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:12,554 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:44:12,554 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:44:12,554 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:12,554 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:44:12,929 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:44:12,929 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:44:13,195 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/vuqz7cpd-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:44:13,314 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:13,930 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:14,315 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:15,324 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:15,930 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:16,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:17,315 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:17,636 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:17,931 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:18,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:19,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:20,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:21,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:21,932 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:22,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:23,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:23,316 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:24,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:24,949 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:44:25,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:44:25,273 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:44:25,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:26,317 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:27,319 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:28,319 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:28,342 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:29,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:30,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:31,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:32,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:33,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:34,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:34,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:35,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:36,320 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:37,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:38,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:39,281 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:39,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:40,273 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:44:40,274 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:44:40,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:41,321 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:42,322 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:43,322 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:44,327 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:44,717 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:44,892 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:44:44,893 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 with policy now +2024-12-31 22:44:44,901 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:44:44,902 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:44:44,902 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:44:44,902 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:44:44,940 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:44:44,940 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 +2024-12-31 22:44:44,941 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:44:45,157 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/cna1il3e-media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 +2024-12-31 22:44:45,322 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:45,455 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/jsmpnc8i-media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 +2024-12-31 22:44:45,941 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:46,326 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:46,941 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:47,326 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:48,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:49,332 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:49,942 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:50,291 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:50,348 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:51,341 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:52,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:53,333 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:53,943 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:54,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:54,950 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:44:55,279 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:44:55,279 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:44:55,341 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:55,440 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:44:56,331 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:56,608 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:44:56,608 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:44:56,610 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:44:56,944 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:44:56,944 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:44:57,333 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:57,945 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:44:58,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:59,338 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:44:59,945 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:00,339 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:00,449 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:01,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:01,946 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:02,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:03,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:04,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:05,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:06,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:06,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:07,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:08,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:09,334 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:10,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:45:10,274 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:45:10,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:11,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:11,382 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:12,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:13,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:14,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:15,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:16,335 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:17,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:17,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:18,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:19,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:20,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:21,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:22,284 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:22,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:23,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:24,336 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:24,950 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:45:25,274 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:45:25,274 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:45:25,337 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:26,337 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:27,337 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:27,371 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:28,341 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:28,649 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:45:28,649 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 with policy now +2024-12-31 22:45:28,651 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:45:28,652 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:45:28,652 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:28,652 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:28,905 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/ss3ni1qi-media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 +2024-12-31 22:45:28,958 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:45:28,959 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 +2024-12-31 22:45:28,959 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval +2024-12-31 22:45:29,230 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/bxmyi9i0-media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 +2024-12-31 22:45:29,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:29,959 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:30,348 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:30,959 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:31,350 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:32,354 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:32,376 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:33,355 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:33,960 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:34,350 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:35,358 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:36,354 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:37,357 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:37,962 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:38,293 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:38,356 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:39,360 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:40,280 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:45:40,281 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:45:40,354 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:40,583 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 22:45:40,583 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 22:45:40,792 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval +2024-12-31 22:45:40,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table +2024-12-31 22:45:41,037 INFO wandb-upload_1:1761088 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp1zpzrit +2024-12-31 22:45:41,187 INFO wandb-upload_2:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/p2vik5j3-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 22:45:41,195 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 22:45:41,204 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:45:41,225 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/73j5lwt0-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:45:41,350 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:41,382 INFO SenderThread:1761088 [sender.py:send_request_log_artifact():1525] logged artifact run-hab6e0c2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE3MjA5MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3NDY5NA==', 'latestArtifact': None}} +2024-12-31 22:45:41,382 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:45:41,382 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 22:45:41,383 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:45:41,383 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,383 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,383 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:45:41,383 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,384 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,384 DEBUG SenderThread:1761088 [sender.py:send():379] send: telemetry +2024-12-31 22:45:41,384 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,385 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,385 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,385 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,387 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,387 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,388 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,388 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,388 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,388 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,388 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,389 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,389 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,389 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,389 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,390 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,390 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,390 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,390 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,390 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,391 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:41,391 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:41,391 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 22:45:41,628 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/qvglvppt-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 22:45:41,728 INFO wandb-upload_0:1761088 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpseruw75k +2024-12-31 22:45:41,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:45:41,963 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:41,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:45:41,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json +2024-12-31 22:45:41,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval +2024-12-31 22:45:42,142 INFO SenderThread:1761088 [sender.py:send_request_log_artifact():1525] logged artifact run-hab6e0c2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE3MjA5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3NDY5Ng==', 'latestArtifact': None}} +2024-12-31 22:45:42,142 DEBUG SenderThread:1761088 [sender.py:send():379] send: files +2024-12-31 22:45:42,142 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json with policy now +2024-12-31 22:45:42,142 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:45:42,142 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:45:42,143 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:42,324 INFO wandb-upload_2:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/tupuxcci-media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json +2024-12-31 22:45:42,350 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:42,579 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/zewlda6l-media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json +2024-12-31 22:45:42,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:45:43,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:43,964 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:45:44,146 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:44,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:44,965 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml +2024-12-31 22:45:45,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:46,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:47,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:48,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:49,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:49,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:50,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:51,351 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:52,352 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:53,352 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:45:54,285 DEBUG SenderThread:1761088 [sender.py:send():379] send: exit +2024-12-31 22:45:54,286 INFO SenderThread:1761088 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 22:45:54,286 INFO SenderThread:1761088 [sender.py:send_exit():588] handling runtime: 269 +2024-12-31 22:45:54,286 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:45:54,286 INFO SenderThread:1761088 [sender.py:send_exit():594] send defer +2024-12-31 22:45:54,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,286 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 22:45:54,286 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:45:54,286 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:45:54,287 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 22:45:54,287 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 1 +2024-12-31 22:45:54,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,287 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 22:45:54,287 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:45:54,287 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 22:45:54,287 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 2 +2024-12-31 22:45:54,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,287 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 22:45:54,287 INFO HandlerThread:1761088 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 22:45:54,287 DEBUG SystemMonitor:1761088 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 22:45:54,287 DEBUG SystemMonitor:1761088 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 22:45:54,288 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 22:45:54,288 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 22:45:54,339 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 22:45:54,339 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 22:45:54,339 INFO HandlerThread:1761088 [interfaces.py:finish():200] Joined network monitor +2024-12-31 22:45:54,340 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:45:54,340 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 22:45:54,340 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 3 +2024-12-31 22:45:54,340 DEBUG SenderThread:1761088 [sender.py:send():379] send: stats +2024-12-31 22:45:54,340 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,340 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 22:45:54,340 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:45:54,340 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 22:45:54,340 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 4 +2024-12-31 22:45:54,340 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:45:54,341 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 22:45:54,969 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:45:55,969 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:46:00,669 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:00,669 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:46:00,669 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 22:46:00,671 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:00,671 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 22:46:00,671 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 5 +2024-12-31 22:46:00,671 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:00,671 DEBUG SenderThread:1761088 [sender.py:send():379] send: history +2024-12-31 22:46:00,671 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:46:00,671 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:00,671 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 22:46:00,672 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:46:00,672 DEBUG SenderThread:1761088 [sender.py:send():379] send: summary +2024-12-31 22:46:00,672 INFO SenderThread:1761088 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:46:00,673 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 6 +2024-12-31 22:46:00,673 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:00,673 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 22:46:00,673 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 7 +2024-12-31 22:46:00,673 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:46:00,673 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:00,673 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 22:46:00,673 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:00,673 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 22:46:00,971 INFO Thread-12 :1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:46:01,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:02,523 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 8 +2024-12-31 22:46:02,523 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:02,524 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:02,524 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 22:46:02,524 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:02,524 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 22:46:02,524 INFO SenderThread:1761088 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 22:46:02,526 INFO SenderThread:1761088 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 22:46:02,528 INFO SenderThread:1761088 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 22:46:02,538 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 9 +2024-12-31 22:46:02,539 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:02,539 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 22:46:02,539 DEBUG SenderThread:1761088 [sender.py:send():379] send: artifact +2024-12-31 22:46:02,725 INFO SenderThread:1761088 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTY3NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzY3NQ==', 'versionIndex': 398}}} +2024-12-31 22:46:02,726 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:02,726 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 22:46:02,726 INFO SenderThread:1761088 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 22:46:02,972 INFO SenderThread:1761088 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml config.yaml +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json wandb-summary.json +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:46:02,973 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/requirements.txt requirements.txt +2024-12-31 22:46:02,976 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-metadata.json wandb-metadata.json +2024-12-31 22:46:02,976 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log output.log +2024-12-31 22:46:02,976 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/diff.patch diff.patch +2024-12-31 22:46:02,977 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 media/videos/eval/4_eval_4_2efe40fe909e07fe4bc5.mp4 +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 media/videos/eval/1_eval_1_18c2b938cd7782cd9f25.mp4 +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 media/videos/eval/2_eval_2_994bf0b5346b4ff89086.mp4 +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 media/videos/eval/3_eval_3_e78024cdb724dea405d6.mp4 +2024-12-31 22:46:02,981 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 media/videos/eval/0_eval_0_0a609f5ccf30787a62b5.mp4 +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json media/table/eval/total_success_rate_ci_6_aa5e6a1bd4b6c595ca4a.table.json +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 10 +2024-12-31 22:46:02,982 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:02,982 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 22:46:02,982 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 22:46:02,982 INFO SenderThread:1761088 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 22:46:03,170 INFO wandb-upload_1:1761088 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/wandb-summary.json +2024-12-31 22:46:03,172 INFO wandb-upload_2:1761088 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/config.yaml +2024-12-31 22:46:03,192 INFO wandb-upload_4:1761088 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/output.log +2024-12-31 22:46:03,222 INFO wandb-upload_0:1761088 [upload_job.py:push():130] Uploaded file /tmp/tmplxtlnjw8wandb/fg1qo4kf-events.out.tfevents.1735702885.leonmkim-ROG-Strix-G15CS-G15CS.1760903.0 +2024-12-31 22:46:03,241 INFO wandb-upload_3:1761088 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/files/requirements.txt +2024-12-31 22:46:03,287 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:03,288 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:03,441 INFO Thread-11 (_thread_body):1761088 [sender.py:transition_state():614] send defer: 11 +2024-12-31 22:46:03,441 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:03,441 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 22:46:03,442 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:03,442 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 22:46:03,442 INFO SenderThread:1761088 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 22:46:03,443 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 12 +2024-12-31 22:46:03,443 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:03,443 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 22:46:03,443 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:03,443 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 22:46:03,443 INFO SenderThread:1761088 [file_stream.py:finish():601] file stream finish called +2024-12-31 22:46:03,618 INFO SenderThread:1761088 [file_stream.py:finish():605] file stream finish is done +2024-12-31 22:46:03,618 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 13 +2024-12-31 22:46:03,618 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:03,618 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 22:46:03,618 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:03,618 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 22:46:03,618 INFO SenderThread:1761088 [sender.py:transition_state():614] send defer: 14 +2024-12-31 22:46:03,618 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:46:03,618 INFO HandlerThread:1761088 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 22:46:03,619 DEBUG SenderThread:1761088 [sender.py:send():379] send: final +2024-12-31 22:46:03,619 DEBUG SenderThread:1761088 [sender.py:send():379] send: footer +2024-12-31 22:46:03,619 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: defer +2024-12-31 22:46:03,619 INFO SenderThread:1761088 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 22:46:03,619 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:03,619 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 22:46:03,620 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:03,620 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:46:03,620 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: server_info +2024-12-31 22:46:03,620 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 22:46:03,622 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 22:46:03,623 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:46:03,683 DEBUG SenderThread:1761088 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:46:03,683 INFO MainThread:1761088 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 22:46:03,683 INFO MainThread:1761088 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 22:46:03,684 INFO MainThread:1761088 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 22:46:03,684 DEBUG HandlerThread:1761088 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 22:46:03,684 INFO HandlerThread:1761088 [handler.py:finish():882] shutting down handler +2024-12-31 22:46:04,621 INFO WriterThread:1761088 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/run-hab6e0c2.wandb +2024-12-31 22:46:04,683 INFO SenderThread:1761088 [sender.py:finish():1615] shutting down sender +2024-12-31 22:46:04,683 INFO SenderThread:1761088 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 22:46:04,683 INFO SenderThread:1761088 [file_pusher.py:join():175] waiting for file pusher diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug.log b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f38d3004666972414e1ba60808835a50ad4c7884 --- /dev/null +++ b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Configure stats pid to 1760903 +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/settings +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 22:41:24,644 INFO MainThread:1760903 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug.log +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/logs/debug-internal.log +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:init():569] calling init triggers +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2'} +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:init():619] starting backend +2024-12-31 22:41:24,645 INFO MainThread:1760903 [wandb_init.py:init():623] setting up manager +2024-12-31 22:41:24,650 INFO MainThread:1760903 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 22:41:24,651 INFO MainThread:1760903 [wandb_init.py:init():631] backend started and connected +2024-12-31 22:41:24,664 INFO MainThread:1760903 [wandb_init.py:init():720] updated telemetry +2024-12-31 22:41:24,670 INFO MainThread:1760903 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 22:41:24,891 INFO MainThread:1760903 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 22:41:24,932 INFO MainThread:1760903 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 22:41:24,932 INFO MainThread:1760903 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 22:41:25,268 INFO MainThread:1760903 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 22:41:25,269 INFO MainThread:1760903 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 22:41:25,269 INFO MainThread:1760903 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 22:41:25,269 INFO MainThread:1760903 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 22:41:25,270 INFO MainThread:1760903 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 22:41:25,270 INFO MainThread:1760903 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/hab6e0c2/tb, True +2024-12-31 22:41:29,284 INFO MainThread:1760903 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2024-12-31 22:46:04,685 WARNING MsgRouterThr:1760903 [router.py:message_loop():77] message_loop has been closed diff --git a/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/run-hab6e0c2.wandb b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/run-hab6e0c2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e3aa988286adeaed26fb791137eab16429f5cb33 Binary files /dev/null and b/hab6e0c2/wandb/run-20241231_224124-hab6e0c2/run-hab6e0c2.wandb differ diff --git a/nvbqdprb/.hydra/config.yaml b/nvbqdprb/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/nvbqdprb/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/nvbqdprb/.hydra/hydra.yaml b/nvbqdprb/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0fe2d9cbb100b7ab786f13b60967974fcc600d5 --- /dev/null +++ b/nvbqdprb/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/nvbqdprb + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/nvbqdprb/.hydra/overrides.yaml b/nvbqdprb/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/nvbqdprb/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/nvbqdprb/eval_policy.log b/nvbqdprb/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..52c3dd5dfd4d126cf303c6da24407cbcb475e7cb --- /dev/null +++ b/nvbqdprb/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-31 22:41:16,715][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 22:41:16,718][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-31 22:41:20,120][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/qo464509/.hydra/config.yaml b/qo464509/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe --- /dev/null +++ b/qo464509/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1007_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/qo464509/.hydra/hydra.yaml b/qo464509/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83cf27697e5e0214cbfe44bb647945c07870406e --- /dev/null +++ b/qo464509/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/qo464509/.hydra/overrides.yaml b/qo464509/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/qo464509/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/qo464509/episode_rosbags/aligned_depth_to_color_K.npy b/qo464509/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/qo464509/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/qo464509/episode_rosbags/cam_tf_world.npy b/qo464509/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/qo464509/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/qo464509/episode_rosbags/color_K.npy b/qo464509/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/qo464509/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/qo464509/episode_rosbags/depth_K.npy b/qo464509/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/qo464509/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/qo464509/eval_robot.log b/qo464509/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..2d2110d2da045263b2886443630fbc1f4f7d9668 --- /dev/null +++ b/qo464509/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 10:51:37,650][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 10:51:37,653][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/qo464509/eval_video/0_eval.mp4 b/qo464509/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a80b6ce52cfbe4e2685a7d59db4e40506fd65ab --- /dev/null +++ b/qo464509/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:047a5015a7283e01ecf846328ae9657f7a1c9c6363711c088b07dca8d38bb58e +size 1171433 diff --git a/qo464509/eval_video/1_eval.mp4 b/qo464509/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e998ffbc01dcd123a75a71068d628923a7718e2 --- /dev/null +++ b/qo464509/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:561430e6869e3e143edff69a33a0a5419490da9dc2cd62a37cd3c8dd1e103138 +size 1155133 diff --git a/qo464509/eval_video/2_eval.mp4 b/qo464509/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fe48885d87de51b5b9010d0def36e9325cecf0d2 --- /dev/null +++ b/qo464509/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89284451a45c1f0bdd58c411b3f2746f9a9e0cc380861b329ac65541ac9a014d +size 1169659 diff --git a/qo464509/eval_video/3_eval.mp4 b/qo464509/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dac2b086d5d4fb5c8c90902b3c421673747a0f2c --- /dev/null +++ b/qo464509/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5ed70decb9e079021258fd852075e70889c82908ba8c172a05c9cc64096dec1 +size 1180706 diff --git a/qo464509/eval_video/4_eval.mp4 b/qo464509/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7298dc698897a3d19847d7d6f87c0f50be092f3 --- /dev/null +++ b/qo464509/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f9d89dde9cf486c83fda6962a9a9987b329eb4879b6e8369851702942f41389 +size 1196204 diff --git a/qo464509/tb/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 b/qo464509/tb/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 new file mode 100644 index 0000000000000000000000000000000000000000..0fd9f568140280da359b96a52612e1f7252e2485 --- /dev/null +++ b/qo464509/tb/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f75a1868bf09cd007721ad28247e17c9b3ec57fb379973f612dc6794169d89fc +size 6539 diff --git a/qo464509/wandb/debug-internal.log b/qo464509/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..85dacf5fee9d780d35335c7fdc2a1578fb4c8a7d --- /dev/null +++ b/qo464509/wandb/debug-internal.log @@ -0,0 +1,835 @@ +2025-01-01 10:51:43,767 INFO StreamThr :1786163 [internal.py:wandb_internal():85] W&B internal server running at pid: 1786163, started at: 2025-01-01 10:51:43.766599 +2025-01-01 10:51:43,768 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status +2025-01-01 10:51:43,779 INFO WriterThread:1786163 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/run-qo464509.wandb +2025-01-01 10:51:43,780 DEBUG SenderThread:1786163 [sender.py:send():379] send: header +2025-01-01 10:51:43,787 DEBUG SenderThread:1786163 [sender.py:send():379] send: run +2025-01-01 10:51:44,066 INFO SenderThread:1786163 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files +2025-01-01 10:51:44,066 INFO SenderThread:1786163 [sender.py:_start_run_threads():1188] run started: qo464509 with start time 1735746703.767215 +2025-01-01 10:51:44,083 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 10:51:44,084 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: check_version +2025-01-01 10:51:44,159 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 10:51:44,164 DEBUG HandlerThread:1786163 [system_info.py:__init__():26] System info init +2025-01-01 10:51:44,164 DEBUG HandlerThread:1786163 [system_info.py:__init__():41] System info init done +2025-01-01 10:51:44,164 INFO HandlerThread:1786163 [system_monitor.py:start():194] Starting system monitor +2025-01-01 10:51:44,164 INFO SystemMonitor:1786163 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 10:51:44,164 INFO HandlerThread:1786163 [system_monitor.py:probe():214] Collecting system info +2025-01-01 10:51:44,165 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 10:51:44,166 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started disk monitoring +2025-01-01 10:51:44,166 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 10:51:44,167 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started memory monitoring +2025-01-01 10:51:44,167 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started network monitoring +2025-01-01 10:51:44,177 DEBUG HandlerThread:1786163 [system_info.py:probe():152] Probing system +2025-01-01 10:51:44,179 DEBUG HandlerThread:1786163 [system_info.py:_probe_git():137] Probing git +2025-01-01 10:51:44,183 DEBUG HandlerThread:1786163 [system_info.py:_probe_git():145] Probing git done +2025-01-01 10:51:44,183 DEBUG HandlerThread:1786163 [system_info.py:probe():200] Probing system done +2025-01-01 10:51:44,183 DEBUG HandlerThread:1786163 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T15:51:44.177372', 'startedAt': '2025-01-01T15:51:43.758070', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.66475, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.988, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.309, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 650.9016075134277}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 10:51:44,183 INFO HandlerThread:1786163 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 10:51:44,184 INFO HandlerThread:1786163 [system_monitor.py:probe():227] Publishing system info +2025-01-01 10:51:44,184 DEBUG HandlerThread:1786163 [system_info.py:_save_code():44] Saving code +2025-01-01 10:51:44,185 DEBUG HandlerThread:1786163 [system_info.py:_save_code():67] Saving code done +2025-01-01 10:51:44,185 DEBUG HandlerThread:1786163 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 10:51:44,255 DEBUG HandlerThread:1786163 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 10:51:44,255 INFO HandlerThread:1786163 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 10:51:44,259 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:51:44,259 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 10:51:44,259 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 10:51:44,259 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 10:51:44,534 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/3ixjw6p7-diff.patch +2025-01-01 10:51:44,538 INFO wandb-upload_0:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/sp2l1kdn-wandb-metadata.json +2025-01-01 10:51:44,542 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/m0ho2arv-code/FISH/eval_robot.py +2025-01-01 10:51:44,547 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 10:51:44,547 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: python_packages +2025-01-01 10:51:44,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:44,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:51:44,549 DEBUG SenderThread:1786163 [sender.py:send():379] send: telemetry +2025-01-01 10:51:44,549 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:51:44,597 INFO HandlerThread:1786163 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/tb" + save: true +} +control { + relay_id: "3d2e6f45d36949e79d82444785edfe6c" +} +_info { + stream_id: "qo464509" +} + +2025-01-01 10:51:44,597 DEBUG HandlerThread:1786163 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 10:51:44,676 DEBUG SenderThread:1786163 [sender.py:send():379] send: tbrecord +2025-01-01 10:51:44,677 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:51:44,677 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 with policy live +2025-01-01 10:51:44,861 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/rng1gizz-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/diff.patch +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/code/FISH/eval_robot.py +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/requirements.txt +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-metadata.json +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/code +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/code/FISH +2025-01-01 10:51:45,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:46,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:47,067 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:51:47,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:48,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:49,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:51:49,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:50,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:51,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:52,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:53,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:54,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:51:54,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:55,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:56,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:57,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:58,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:59,547 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:51:59,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:51:59,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:59,675 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:00,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:01,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:02,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:03,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:04,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:05,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:05,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:06,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:07,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:08,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:09,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:10,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:10,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:11,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:12,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:13,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:14,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:52:14,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:52:14,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:15,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:15,610 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:16,074 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml +2025-01-01 10:52:16,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:17,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:18,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:19,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:20,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:21,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:21,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:22,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:23,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:24,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:25,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:26,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:27,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:27,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:28,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:29,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:52:29,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:52:29,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:30,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:31,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:32,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:32,685 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:33,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:34,331 DEBUG SenderThread:1786163 [sender.py:send():379] send: config +2025-01-01 10:52:34,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:35,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:36,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:37,080 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:37,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:38,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:38,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:39,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:40,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:41,081 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:41,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:42,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:43,082 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:43,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:43,755 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:44,168 DEBUG SystemMonitor:1786163 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 10:52:44,169 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:52:44,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:52:44,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:52:44,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:45,082 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:45,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:46,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:47,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:48,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:48,768 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:49,083 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml +2025-01-01 10:52:49,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:50,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:51,084 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:51,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:52,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:53,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:54,522 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:54,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:55,085 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:55,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:56,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:57,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:58,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:59,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:52:59,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:52:59,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:59,687 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:00,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:01,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:02,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:03,087 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:03,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:04,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:05,088 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:05,089 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:05,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:06,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:07,089 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:07,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:08,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:09,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:10,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:10,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:11,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:12,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:13,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:14,169 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:53:14,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:53:14,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:53:14,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:15,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:15,614 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:16,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:17,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:18,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:19,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:20,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:20,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:21,093 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:21,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:22,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:23,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:24,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:25,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:26,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:26,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:27,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:28,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:29,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:53:29,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:53:29,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:30,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:31,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:31,679 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:32,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:33,096 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:33,158 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:53:33,159 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 with policy now +2025-01-01 10:53:33,159 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:53:33,160 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:53:33,160 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:53:33,162 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:53:33,387 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/2mla3ytc-media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 +2025-01-01 10:53:33,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:34,096 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 +2025-01-01 10:53:34,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:53:34,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:53:34,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media +2025-01-01 10:53:34,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos +2025-01-01 10:53:34,401 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/dg1j6msb-media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 +2025-01-01 10:53:34,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:35,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:35,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:36,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:37,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:37,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:38,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:39,098 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:39,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:40,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:41,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:42,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:43,100 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:43,100 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:53:43,116 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:43,351 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/32fstrdh-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:53:43,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:44,169 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:53:44,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:53:44,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:53:44,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:45,100 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:45,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:46,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:47,101 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:47,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:48,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:49,071 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:49,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:50,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:51,564 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:52,564 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:53,564 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:54,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:54,564 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:55,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:56,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:57,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:58,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:59,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:53:59,550 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:53:59,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:59,610 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:00,572 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:01,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:02,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:03,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:04,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:05,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:05,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:06,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:07,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:08,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:09,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:10,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:10,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:11,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:12,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:13,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:14,170 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:54:14,288 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:54:14,288 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 with policy now +2025-01-01 10:54:14,289 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:54:14,290 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:54:14,290 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:54:14,290 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:54:14,514 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/zf34tc2c-media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 +2025-01-01 10:54:14,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:54:14,549 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:54:14,570 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:15,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:15,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:54:15,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 +2025-01-01 10:54:15,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:54:15,378 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/tao80fiw-media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 +2025-01-01 10:54:15,574 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:15,627 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:16,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:16,577 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:17,577 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:18,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:19,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:20,576 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:21,112 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:21,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:21,587 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:22,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:23,112 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:23,113 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:54:23,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:24,099 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:54:24,099 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:54:24,099 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:54:24,113 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:54:24,577 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:25,113 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:25,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:26,577 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:26,623 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:27,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:28,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:29,115 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:29,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:54:29,549 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:54:29,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:30,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:31,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:31,700 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:32,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:33,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:34,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:35,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:36,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:37,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:37,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:38,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:39,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:40,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:41,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:42,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:42,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:43,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:44,170 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:54:44,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:54:44,549 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:54:44,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:45,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:46,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:47,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:47,636 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:48,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:49,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:50,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:51,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:52,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:53,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:53,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:54,582 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:54,925 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:54:54,926 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:54:54,926 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 with policy now +2025-01-01 10:54:54,926 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:54:54,926 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:54:54,926 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:54:55,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:55,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:54:55,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 +2025-01-01 10:54:55,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:54:55,174 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/8aa7ymy1-media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 +2025-01-01 10:54:55,417 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/gtz2io06-media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 +2025-01-01 10:54:55,585 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:56,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:56,591 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:57,124 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:57,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:58,590 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:58,960 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:59,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:54:59,555 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:54:59,601 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:00,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:01,125 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:01,602 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:02,590 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:03,592 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:04,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:04,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:05,126 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:05,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:06,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:07,595 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:08,599 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:09,600 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:10,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:10,597 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:11,597 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:12,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:13,601 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:14,171 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:55:14,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:55:14,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:55:14,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:14,922 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:55:14,922 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:55:14,922 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:55:15,130 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:55:15,130 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:15,130 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:55:15,364 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/2zbmys5t-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:55:15,602 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:16,249 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:16,604 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:17,130 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:17,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:18,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:19,131 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:19,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:20,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:21,132 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:21,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:21,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:22,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:23,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:24,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:25,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:26,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:26,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:27,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:28,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:29,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:55:29,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:55:29,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:30,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:31,135 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:31,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:31,681 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:32,608 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:33,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:34,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:35,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:36,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:37,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:37,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:38,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:39,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:40,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:41,609 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:42,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:42,608 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:43,608 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:44,171 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:55:44,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:55:44,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:55:44,608 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:45,609 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:46,337 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:55:46,337 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 with policy now +2025-01-01 10:55:46,342 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:55:46,343 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:55:46,343 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:55:46,343 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:55:46,547 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/ozvfccbj-media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 +2025-01-01 10:55:46,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:47,141 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:47,141 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:55:47,141 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 +2025-01-01 10:55:47,141 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:55:47,403 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/q58ldh30-media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 +2025-01-01 10:55:47,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:48,375 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:48,613 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:49,614 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:50,616 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:51,615 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:52,630 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:53,143 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:53,615 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:53,635 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:54,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:54,970 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:55:54,970 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:55:54,970 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:55:55,143 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:55:55,144 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:55,144 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:55:55,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:56,623 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:57,144 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:57,618 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:58,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:58,669 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:59,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:55:59,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:55:59,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:00,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:01,146 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:01,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:02,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:03,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:04,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:04,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:05,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:06,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:07,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:08,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:09,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:09,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:10,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:11,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:12,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:13,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:14,171 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:56:14,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:56:14,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:56:14,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:14,679 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:15,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:16,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:17,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:18,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:19,152 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:19,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:20,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:20,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:21,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:22,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:23,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:24,623 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:25,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:25,623 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:26,630 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:26,990 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:56:26,990 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 with policy now +2025-01-01 10:56:26,995 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:56:26,996 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:56:26,996 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:26,996 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:27,154 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:27,154 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:27,155 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 +2025-01-01 10:56:27,155 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:56:27,210 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/w1vs6v9d-media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 +2025-01-01 10:56:27,508 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/yclzgvqr-media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 +2025-01-01 10:56:27,645 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:28,630 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:29,155 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:29,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:56:29,563 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:56:29,631 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:30,637 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:30,646 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:31,635 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:32,634 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:33,156 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:33,670 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:34,657 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:35,650 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:35,710 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:36,643 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:37,158 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:37,648 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:38,663 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:39,649 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:40,656 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:41,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:41,646 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:42,652 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:43,656 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:44,172 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:56:44,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:56:44,556 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:56:44,654 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:45,654 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:46,648 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:46,664 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:47,657 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:48,661 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:49,683 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:50,035 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:56:50,035 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:50,035 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:50,162 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:56:50,162 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:50,246 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 10:56:50,247 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 10:56:50,388 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:56:50,399 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/3qtsnd0z-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:56:50,659 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:50,671 INFO wandb-upload_2:1786163 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1r6scxd5 +2025-01-01 10:56:51,022 INFO SenderThread:1786163 [sender.py:send_request_log_artifact():1525] logged artifact run-qo464509-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY2Njg5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTIzOTEyMQ==', 'latestArtifact': None}} +2025-01-01 10:56:51,022 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:56:51,022 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 10:56:51,023 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:56:51,023 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:51,023 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:51,162 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval +2025-01-01 10:56:51,227 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/xj491i9b-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 10:56:51,488 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/r93183fk-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 10:56:51,660 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:52,024 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:52,660 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:53,428 DEBUG SenderThread:1786163 [sender.py:send():379] send: telemetry +2025-01-01 10:56:53,428 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,429 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,430 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,430 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,430 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,431 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,435 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,435 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,436 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,436 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,441 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,441 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,441 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,442 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,448 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,449 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,449 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,450 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,456 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,456 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,457 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,458 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,537 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 10:56:53,538 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 10:56:53,547 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:56:53,660 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:53,889 INFO wandb-upload_1:1786163 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwb2orbjw +2025-01-01 10:56:54,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:56:54,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:54,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json +2025-01-01 10:56:54,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval +2025-01-01 10:56:54,287 INFO SenderThread:1786163 [sender.py:send_request_log_artifact():1525] logged artifact run-qo464509-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NjkzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTIzOTEyNw==', 'latestArtifact': None}} +2025-01-01 10:56:54,287 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:56:54,287 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json with policy now +2025-01-01 10:56:54,288 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:56:54,289 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:54,290 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:54,461 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/et2enguc-media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json +2025-01-01 10:56:54,660 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:54,695 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/93m5r6pj-media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json +2025-01-01 10:56:55,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:55,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:55,361 DEBUG SenderThread:1786163 [sender.py:send():379] send: exit +2025-01-01 10:56:55,361 INFO SenderThread:1786163 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:send_exit():588] handling runtime: 311 +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:send_exit():594] send defer +2025-01-01 10:56:55,362 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,362 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 10:56:55,362 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 1 +2025-01-01 10:56:55,363 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,363 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 10:56:55,363 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:56:55,363 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 10:56:55,363 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 2 +2025-01-01 10:56:55,363 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,363 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 10:56:55,363 INFO HandlerThread:1786163 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 10:56:55,363 DEBUG SystemMonitor:1786163 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 10:56:55,363 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 10:56:55,364 DEBUG SystemMonitor:1786163 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 10:56:55,364 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 10:56:55,415 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 10:56:55,415 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 10:56:55,415 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined network monitor +2025-01-01 10:56:55,416 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:56:55,416 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 10:56:55,416 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 3 +2025-01-01 10:56:55,416 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:56:55,416 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,416 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 10:56:55,416 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:56:55,417 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 10:56:55,417 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 4 +2025-01-01 10:56:55,417 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,417 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 10:56:56,165 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:57,165 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:57:02,044 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:02,044 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:57:02,045 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 10:57:02,046 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:02,046 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 10:57:02,046 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 5 +2025-01-01 10:57:02,046 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:02,046 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:02,046 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:57:02,046 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 10:57:02,046 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:57:02,047 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:57:02,048 DEBUG SenderThread:1786163 [sender.py:send():379] send: summary +2025-01-01 10:57:02,051 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:57:02,051 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:02,052 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 10:57:02,052 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 6 +2025-01-01 10:57:02,052 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:02,052 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 10:57:02,052 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:02,052 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 10:57:02,055 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:57:02,166 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 7 +2025-01-01 10:57:02,166 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:02,166 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 10:57:02,167 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:02,167 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 10:57:02,167 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml +2025-01-01 10:57:02,167 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:57:02,362 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:04,074 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 8 +2025-01-01 10:57:04,074 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:04,074 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:04,075 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 10:57:04,075 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:04,075 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 10:57:04,075 INFO SenderThread:1786163 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 10:57:04,077 INFO SenderThread:1786163 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 10:57:04,079 INFO SenderThread:1786163 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 10:57:04,093 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 9 +2025-01-01 10:57:04,093 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:04,093 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 10:57:04,093 DEBUG SenderThread:1786163 [sender.py:send():379] send: artifact +2025-01-01 10:57:04,168 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:57:04,363 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:04,632 INFO wandb-upload_1:1786163 [upload_job.py:push():85] Skipped uploading /tmp/tmpqfdli0iz/wandb-job.json +2025-01-01 10:57:04,633 INFO wandb-upload_0:1786163 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmprokw9jhr +2025-01-01 10:57:04,768 INFO wandb-upload_2:1786163 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8m6d6qiy +2025-01-01 10:57:05,264 INFO SenderThread:1786163 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'versionIndex': 403}}} +2025-01-01 10:57:05,264 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:05,264 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 10:57:05,264 INFO SenderThread:1786163 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 10:57:06,169 INFO SenderThread:1786163 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files +2025-01-01 10:57:06,169 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml config.yaml +2025-01-01 10:57:06,169 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json wandb-summary.json +2025-01-01 10:57:06,169 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:57:06,171 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/requirements.txt requirements.txt +2025-01-01 10:57:06,171 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-metadata.json wandb-metadata.json +2025-01-01 10:57:06,171 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log output.log +2025-01-01 10:57:06,171 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/diff.patch diff.patch +2025-01-01 10:57:06,175 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json +2025-01-01 10:57:06,181 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 10:57:06,181 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 10 +2025-01-01 10:57:06,181 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:06,181 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,181 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 10:57:06,181 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,181 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 10:57:06,181 INFO SenderThread:1786163 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 10:57:06,344 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml +2025-01-01 10:57:06,360 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:57:06,364 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:06,364 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:06,401 INFO wandb-upload_3:1786163 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/requirements.txt +2025-01-01 10:57:06,410 INFO wandb-upload_4:1786163 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:57:06,443 INFO wandb-upload_0:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/uxvw1l74-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:57:06,644 INFO Thread-11 (_thread_body):1786163 [sender.py:transition_state():614] send defer: 11 +2025-01-01 10:57:06,644 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,644 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 10:57:06,644 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,644 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 10:57:06,644 INFO SenderThread:1786163 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 10:57:06,645 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 12 +2025-01-01 10:57:06,645 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,645 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 10:57:06,646 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,646 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 10:57:06,646 INFO SenderThread:1786163 [file_stream.py:finish():601] file stream finish called +2025-01-01 10:57:06,839 INFO SenderThread:1786163 [file_stream.py:finish():605] file stream finish is done +2025-01-01 10:57:06,839 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 13 +2025-01-01 10:57:06,839 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,839 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 10:57:06,839 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,839 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 10:57:06,839 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 14 +2025-01-01 10:57:06,840 DEBUG SenderThread:1786163 [sender.py:send():379] send: final +2025-01-01 10:57:06,840 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,840 DEBUG SenderThread:1786163 [sender.py:send():379] send: footer +2025-01-01 10:57:06,840 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 10:57:06,840 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,840 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 10:57:06,840 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:06,841 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 10:57:06,841 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:06,841 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:06,842 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: server_info +2025-01-01 10:57:06,843 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 10:57:06,844 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 10:57:06,845 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:57:06,896 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:06,896 INFO MainThread:1786163 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 10:57:06,897 INFO MainThread:1786163 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 10:57:06,897 INFO MainThread:1786163 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 10:57:06,897 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 10:57:06,897 INFO HandlerThread:1786163 [handler.py:finish():882] shutting down handler +2025-01-01 10:57:07,841 INFO WriterThread:1786163 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/run-qo464509.wandb +2025-01-01 10:57:07,896 INFO SenderThread:1786163 [sender.py:finish():1615] shutting down sender +2025-01-01 10:57:07,897 INFO SenderThread:1786163 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 10:57:07,897 INFO SenderThread:1786163 [file_pusher.py:join():175] waiting for file pusher diff --git a/qo464509/wandb/debug.log b/qo464509/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..77cb70a24ba6ece91528a05a44cd1ba863e540e1 --- /dev/null +++ b/qo464509/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Configure stats pid to 1785977 +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/settings +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/logs/debug.log +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/logs/debug-internal.log +2025-01-01 10:51:43,760 INFO MainThread:1785977 [wandb_init.py:init():569] calling init triggers +2025-01-01 10:51:43,760 INFO MainThread:1785977 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509'} +2025-01-01 10:51:43,760 INFO MainThread:1785977 [wandb_init.py:init():619] starting backend +2025-01-01 10:51:43,760 INFO MainThread:1785977 [wandb_init.py:init():623] setting up manager +2025-01-01 10:51:43,765 INFO MainThread:1785977 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 10:51:43,767 INFO MainThread:1785977 [wandb_init.py:init():631] backend started and connected +2025-01-01 10:51:43,779 INFO MainThread:1785977 [wandb_init.py:init():720] updated telemetry +2025-01-01 10:51:43,785 INFO MainThread:1785977 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 10:51:44,083 INFO MainThread:1785977 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 10:51:44,154 INFO MainThread:1785977 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 10:51:44,154 INFO MainThread:1785977 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 10:51:44,547 INFO MainThread:1785977 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 10:51:44,547 INFO MainThread:1785977 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 10:51:44,548 INFO MainThread:1785977 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 10:51:44,548 INFO MainThread:1785977 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 10:51:44,550 INFO MainThread:1785977 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 10:51:44,551 INFO MainThread:1785977 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/tb, True +2025-01-01 10:52:34,330 INFO MainThread:1785977 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-01 10:57:07,898 WARNING MsgRouterThr:1785977 [router.py:message_loop():77] message_loop has been closed diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/code/FISH/eval_robot.py b/qo464509/wandb/run-20250101_105143-qo464509/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml b/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..321c25c29f9ab966a758638d58c1c2929408b35b --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1007_0' +wandb_notes: + desc: null + value: 1007_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735746703 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/diff.patch b/qo464509/wandb/run-20250101_105143-qo464509/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..eff8de3c864ce1e6f5e6f3000cbf990769adf78e --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/diff.patch @@ -0,0 +1,105 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..b9e3dfe 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history ++# wandb_run_id: '1002_0' ++ ++# all books, 6/20 demos per book ++# RGBD+mask+act history + wandb_run_id: '1007_0' + ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json b/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json new file mode 100644 index 0000000000000000000000000000000000000000..76823b7766897ba67ec6457fa9c4ac262f8abd07 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[19, 35, 0.5428571428571428, 0.34505966308831526, 0.7295440434950458, 0.38328675928065914, 0.7106240987356462, 0.40822155030417184, 0.6725112720978945, 0.42618761213109924, 0.6576757542473631]]} \ No newline at end of file diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a80b6ce52cfbe4e2685a7d59db4e40506fd65ab --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:047a5015a7283e01ecf846328ae9657f7a1c9c6363711c088b07dca8d38bb58e +size 1171433 diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e998ffbc01dcd123a75a71068d628923a7718e2 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:561430e6869e3e143edff69a33a0a5419490da9dc2cd62a37cd3c8dd1e103138 +size 1155133 diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fe48885d87de51b5b9010d0def36e9325cecf0d2 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89284451a45c1f0bdd58c411b3f2746f9a9e0cc380861b329ac65541ac9a014d +size 1169659 diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dac2b086d5d4fb5c8c90902b3c421673747a0f2c --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5ed70decb9e079021258fd852075e70889c82908ba8c172a05c9cc64096dec1 +size 1180706 diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7298dc698897a3d19847d7d6f87c0f50be092f3 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f9d89dde9cf486c83fda6962a9a9987b329eb4879b6e8369851702942f41389 +size 1196204 diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/output.log b/qo464509/wandb/run-20250101_105143-qo464509/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f5f7410abd84aad7b42390fd453d78c5a8b79fe1 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/output.log @@ -0,0 +1,279 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735746754.331422]: resetting environment +[INFO] [1735746754.334296]: cleared current plan +[INFO] [1735746754.335164]: moving to home +Move to target timeout: 5.099948406219482. Trans error: 0.005135520566715331. Rot error: 1.3740456569883785 +[INFO] [1735746761.039276]: reached home +[INFO] [1735746761.040642]: reset action history +[INFO] [1735746763.754535]: environment reset +Starting episode 0 +[INFO] [1735746763.756539]: resetting environment +[INFO] [1735746763.761803]: cleared current plan +[INFO] [1735746763.762888]: moving to home +Move to target timeout: 5.000603437423706. Trans error: 0.0051069723188783825. Rot error: 1.3635650353363082 +[INFO] [1735746769.765276]: reached home +[INFO] [1735746769.766871]: reset action history +[INFO] [1735746772.474131]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735746781.371170]: resetting environment +[INFO] [1735746781.375255]: cleared current plan +[INFO] [1735746781.376148]: moving to home +[INFO] [1735746782.378666]: reached home +[INFO] [1735746782.380123]: reset action history +[INFO] [1735746785.088691]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/episode_rosbags/episode_0_2025-01-01-10-53-05.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1785977.log +started roslaunch server http://158.130.50.37:36827/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1786553] +process[print_text-2]: started with pid [1786554] +process[rosbag_record-3]: started with pid [1786556] +started bagging! +For topic gripper_width: timestamp difference is 111660234 for nearest: 1735746799378646604 - target: 1735746799266986370 at idx 186 +For topic gripper_width: timestamp difference is 111825020 for nearest: 1735746799578652174 - target: 1735746799466827154 at idx 187 +[INFO] [1735746812.127310]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735746812.927841]: Stored episode 1. +[INFO] [1735746812.928131]: Saving video... +[INFO] [1735746813.125726]: Video saved! +Starting episode 1 +[INFO] [1735746813.159885]: resetting environment +[INFO] [1735746813.168087]: cleared current plan +[INFO] [1735746813.174012]: moving to home +[INFO] [1735746817.777414]: reached home +[INFO] [1735746817.778122]: reset action history +[INFO] [1735746820.486174]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735746822.112510]: resetting environment +[INFO] [1735746822.114539]: cleared current plan +[INFO] [1735746822.114733]: moving to home +[INFO] [1735746823.116357]: reached home +[INFO] [1735746823.116788]: reset action history +[INFO] [1735746825.822137]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/episode_rosbags/episode_1_2025-01-01-10-53-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1785977.log +started roslaunch server http://158.130.50.37:38531/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1786665] +process[print_text-5]: started with pid [1786666] +process[rosbag_record-6]: started with pid [1786690] +started bagging! +[INFO] [1735746852.766147]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735746853.566571]: Stored episode 2. +[INFO] [1735746853.566861]: Saving video... +[INFO] [1735746854.263509]: Video saved! +Starting episode 2 +[INFO] [1735746854.294565]: resetting environment +[INFO] [1735746854.303530]: cleared current plan +[INFO] [1735746854.305768]: moving to home +[WARN] [1735746853.920311]: Plan exhausted +[WARN] [1735746853.960314]: Plan exhausted +[WARN] [1735746854.000305]: Plan exhausted +[WARN] [1735746854.040316]: Plan exhausted +[WARN] [1735746854.080436]: Plan exhausted +[WARN] [1735746854.121923]: Plan exhausted +[WARN] [1735746854.160820]: Plan exhausted +[WARN] [1735746854.200975]: Plan exhausted +[WARN] [1735746854.240603]: Plan exhausted +[WARN] [1735746854.280423]: Plan exhausted +[INFO] [1735746858.909270]: reached home +[INFO] [1735746858.910736]: reset action history +[INFO] [1735746861.633456]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735746862.903074]: resetting environment +[INFO] [1735746862.911593]: cleared current plan +[INFO] [1735746862.911988]: moving to home +[INFO] [1735746863.915023]: reached home +[INFO] [1735746863.915605]: reset action history +[INFO] [1735746866.623035]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/episode_rosbags/episode_2_2025-01-01-10-54-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1785977.log +started roslaunch server http://158.130.50.37:45061/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1786803] +process[print_text-8]: started with pid [1786804] +process[rosbag_record-9]: started with pid [1786828] +started bagging! +[INFO] [1735746893.590100]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735746894.362610]: Stored episode 3. +[INFO] [1735746894.371259]: Saving video... +[WARN] [1735746894.920773]: Plan exhausted +[INFO] [1735746894.882555]: Video saved! +Starting episode 3 +[INFO] [1735746894.932957]: resetting environment +[INFO] [1735746894.955030]: cleared current plan +[INFO] [1735746894.959998]: moving to home +[INFO] [1735746899.571128]: reached home +[INFO] [1735746899.579472]: reset action history +[INFO] [1735746902.526864]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735746914.201609]: resetting environment +[INFO] [1735746914.204293]: cleared current plan +[INFO] [1735746914.204791]: moving to home +[INFO] [1735746915.206412]: reached home +[INFO] [1735746915.209411]: reset action history +[INFO] [1735746917.933131]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/episode_rosbags/episode_3_2025-01-01-10-55-17.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1785977.log +started roslaunch server http://158.130.50.37:45623/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1786962] +process[print_text-11]: started with pid [1786963] +process[rosbag_record-12]: started with pid [1786964] +started bagging! +For topic gripper_width: timestamp difference is -131363509 for nearest: 1735746918478683116 - target: 1735746918610046625 at idx 9 +For topic gripper_width: timestamp difference is -145143490 for nearest: 1735746928478648443 - target: 1735746928623791933 at idx 132 +[INFO] [1735746945.090283]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735746945.873021]: Stored episode 4. +[INFO] [1735746945.875768]: Saving video... +[INFO] [1735746946.292680]: Video saved! +Starting episode 4 +[INFO] [1735746946.345184]: resetting environment +[INFO] [1735746946.366959]: cleared current plan +[INFO] [1735746946.371711]: moving to home +[INFO] [1735746950.887299]: reached home +[INFO] [1735746950.901662]: reset action history +[INFO] [1735746953.633906]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735746954.870021]: resetting environment +[INFO] [1735746954.875999]: cleared current plan +[INFO] [1735746954.876690]: moving to home +[INFO] [1735746955.878243]: reached home +[INFO] [1735746955.880509]: reset action history +[INFO] [1735746958.668450]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/episode_rosbags/episode_4_2025-01-01-10-55-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1785977.log +started roslaunch server http://158.130.50.37:37063/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1787099] +process[print_text-14]: started with pid [1787100] +process[rosbag_record-15]: started with pid [1787101] +started bagging! +For topic gripper_width: timestamp difference is -243824485 for nearest: 1735746978078644749 - target: 1735746978322469234 at idx 249 +[INFO] [1735746985.812365]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1735746986.603157]: Stored episode 5. +[INFO] [1735746986.603522]: Saving video... +[INFO] [1735746986.932454]: Video saved! +[INFO] [1735746986.996182]: resetting environment +[INFO] [1735746987.014738]: cleared current plan +[INFO] [1735746987.016431]: moving to home +[INFO] [1735746991.724073]: reached home +[INFO] [1735746991.744346]: reset action history +[INFO] [1735746994.609814]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/requirements.txt b/qo464509/wandb/run-20250101_105143-qo464509/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-metadata.json b/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d2063bd8a417c939ab2c8e75b900a01044b58ce8 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T15:51:44.177372", + "startedAt": "2025-01-01T15:51:43.758070", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.66475, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.988, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.309, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 650.9016075134277 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json b/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5b0c75bd698fa6f34f25a3f59a2ee1d3ab75523a --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "047a5015a7283e01ecf846328ae9657f7a1c9c6363711c088b07dca8d38bb58e", "size": 1171433, "path": "media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4"}, "global_step": 664, "_timestamp": 1735747013.457001, "_runtime": 309.6897859573364, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "561430e6869e3e143edff69a33a0a5419490da9dc2cd62a37cd3c8dd1e103138", "size": 1155133, "path": "media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4"}, "eval/num_success": 5.0, "episode": 35.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "89284451a45c1f0bdd58c411b3f2746f9a9e0cc380861b329ac65541ac9a014d", "size": 1169659, "path": "media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a5ed70decb9e079021258fd852075e70889c82908ba8c172a05c9cc64096dec1", "size": 1180706, "path": "media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "0f9d89dde9cf486c83fda6962a9a9987b329eb4879b6e8369851702942f41389", "size": 1196204, "path": "media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://bw6qf1kdbmsd4aqn0aiuczzesxjrsde4z0hxzs0f0w1l3jinsasq8845zirx4dn32dndnrfepji96xmw0j1y9r13ijcpxahur954loi7gljleu91do2hdjtyel1zll3u/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://gkim6tvb56jhhnzvfmyrkn8gk4a81p6m6p5v60buijx97x5cy0wxns5q6k8uce3qhsroksgk0krjnuscpntr0oi36x149fawhgufbs6hsas6o4saul28x5nwu3ga3yuh:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 19, "total_num_episodes": 35, "total_success_rate": 0.5428571428571428, "total_success_rate_lb_0.01": 0.34505966308831526, "total_success_rate_ub_0.01": 0.7295440434950458, "total_success_rate_lb_0.025": 0.38328675928065914, "total_success_rate_ub_0.025": 0.7106240987356462, "total_success_rate_lb_0.05": 0.40822155030417184, "total_success_rate_ub_0.05": 0.6725112720978945, "total_success_rate_lb_0.1": 0.42618761213109924, "total_success_rate_ub_0.1": 0.6576757542473631, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "1a02426d42619cd057a57c1015b0e2ca7ed466650b74e500a5e66f0385de44da", "size": 524, "artifact_path": "wandb-client-artifact://iucp64ucqtlp08gehmekpl2fr7c1yktf9uv6rrvlw55ln6m62jcx9ddqgpd8s8qm0nxg87osl3fud2q7in8egvsuzfhk9zu33zxc6s9ba1gfjq0lrhk450w0ksch8k35/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://b2xenpjjkqt3q1h102mcj6mwmfq1u6dly7sloaxee4vb9598ldslnwprhkvrstemnq08xsqakpqmbd6mk08yr2be2zonxkpv1hapy1xynscq8e82ivcj7wop63scwm2w:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 311}, "eval/total_success_rate": 0.5428571701049805, "eval/total_success_rate_lb_0.01": 0.34505966305732727, "eval/total_success_rate_ub_0.01": 0.7295440435409546, "eval/total_success_rate_lb_0.025": 0.3832867741584778, "eval/total_success_rate_ub_0.025": 0.710624098777771, "eval/total_success_rate_lb_0.05": 0.4082215428352356, "eval/total_success_rate_ub_0.05": 0.6725112795829773, "eval/total_success_rate_lb_0.1": 0.4261876046657562, "eval/total_success_rate_ub_0.1": 0.6576757431030273} \ No newline at end of file diff --git a/qo464509/wandb/run-20250101_105143-qo464509/logs/debug-internal.log b/qo464509/wandb/run-20250101_105143-qo464509/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..85dacf5fee9d780d35335c7fdc2a1578fb4c8a7d --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/logs/debug-internal.log @@ -0,0 +1,835 @@ +2025-01-01 10:51:43,767 INFO StreamThr :1786163 [internal.py:wandb_internal():85] W&B internal server running at pid: 1786163, started at: 2025-01-01 10:51:43.766599 +2025-01-01 10:51:43,768 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status +2025-01-01 10:51:43,779 INFO WriterThread:1786163 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/run-qo464509.wandb +2025-01-01 10:51:43,780 DEBUG SenderThread:1786163 [sender.py:send():379] send: header +2025-01-01 10:51:43,787 DEBUG SenderThread:1786163 [sender.py:send():379] send: run +2025-01-01 10:51:44,066 INFO SenderThread:1786163 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files +2025-01-01 10:51:44,066 INFO SenderThread:1786163 [sender.py:_start_run_threads():1188] run started: qo464509 with start time 1735746703.767215 +2025-01-01 10:51:44,083 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 10:51:44,084 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: check_version +2025-01-01 10:51:44,159 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 10:51:44,164 DEBUG HandlerThread:1786163 [system_info.py:__init__():26] System info init +2025-01-01 10:51:44,164 DEBUG HandlerThread:1786163 [system_info.py:__init__():41] System info init done +2025-01-01 10:51:44,164 INFO HandlerThread:1786163 [system_monitor.py:start():194] Starting system monitor +2025-01-01 10:51:44,164 INFO SystemMonitor:1786163 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 10:51:44,164 INFO HandlerThread:1786163 [system_monitor.py:probe():214] Collecting system info +2025-01-01 10:51:44,165 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 10:51:44,166 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started disk monitoring +2025-01-01 10:51:44,166 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 10:51:44,167 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started memory monitoring +2025-01-01 10:51:44,167 INFO SystemMonitor:1786163 [interfaces.py:start():188] Started network monitoring +2025-01-01 10:51:44,177 DEBUG HandlerThread:1786163 [system_info.py:probe():152] Probing system +2025-01-01 10:51:44,179 DEBUG HandlerThread:1786163 [system_info.py:_probe_git():137] Probing git +2025-01-01 10:51:44,183 DEBUG HandlerThread:1786163 [system_info.py:_probe_git():145] Probing git done +2025-01-01 10:51:44,183 DEBUG HandlerThread:1786163 [system_info.py:probe():200] Probing system done +2025-01-01 10:51:44,183 DEBUG HandlerThread:1786163 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T15:51:44.177372', 'startedAt': '2025-01-01T15:51:43.758070', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.66475, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.988, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.309, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 650.9016075134277}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 10:51:44,183 INFO HandlerThread:1786163 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 10:51:44,184 INFO HandlerThread:1786163 [system_monitor.py:probe():227] Publishing system info +2025-01-01 10:51:44,184 DEBUG HandlerThread:1786163 [system_info.py:_save_code():44] Saving code +2025-01-01 10:51:44,185 DEBUG HandlerThread:1786163 [system_info.py:_save_code():67] Saving code done +2025-01-01 10:51:44,185 DEBUG HandlerThread:1786163 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 10:51:44,255 DEBUG HandlerThread:1786163 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 10:51:44,255 INFO HandlerThread:1786163 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 10:51:44,259 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:51:44,259 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 10:51:44,259 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 10:51:44,259 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 10:51:44,534 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/3ixjw6p7-diff.patch +2025-01-01 10:51:44,538 INFO wandb-upload_0:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/sp2l1kdn-wandb-metadata.json +2025-01-01 10:51:44,542 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/m0ho2arv-code/FISH/eval_robot.py +2025-01-01 10:51:44,547 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 10:51:44,547 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: python_packages +2025-01-01 10:51:44,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:44,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:51:44,549 DEBUG SenderThread:1786163 [sender.py:send():379] send: telemetry +2025-01-01 10:51:44,549 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:51:44,597 INFO HandlerThread:1786163 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/tb" + save: true +} +control { + relay_id: "3d2e6f45d36949e79d82444785edfe6c" +} +_info { + stream_id: "qo464509" +} + +2025-01-01 10:51:44,597 DEBUG HandlerThread:1786163 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 10:51:44,676 DEBUG SenderThread:1786163 [sender.py:send():379] send: tbrecord +2025-01-01 10:51:44,677 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:51:44,677 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 with policy live +2025-01-01 10:51:44,861 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/rng1gizz-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/diff.patch +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/code/FISH/eval_robot.py +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/requirements.txt +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-metadata.json +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/code +2025-01-01 10:51:45,068 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/code/FISH +2025-01-01 10:51:45,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:46,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:47,067 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:51:47,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:48,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:49,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:51:49,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:50,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:51,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:52,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:53,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:54,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:51:54,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:55,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:56,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:57,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:58,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:59,547 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:51:59,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:51:59,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:51:59,675 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:00,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:01,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:02,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:03,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:04,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:05,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:05,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:06,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:07,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:08,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:09,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:10,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:10,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:11,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:12,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:13,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:14,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:52:14,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:52:14,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:15,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:15,610 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:16,074 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml +2025-01-01 10:52:16,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:17,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:18,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:19,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:20,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:21,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:21,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:22,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:23,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:24,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:25,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:26,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:27,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:27,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:28,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:29,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:52:29,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:52:29,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:30,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:31,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:32,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:32,685 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:33,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:34,331 DEBUG SenderThread:1786163 [sender.py:send():379] send: config +2025-01-01 10:52:34,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:35,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:36,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:37,080 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:37,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:38,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:38,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:39,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:40,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:41,081 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:41,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:42,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:43,082 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:43,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:43,755 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:44,168 DEBUG SystemMonitor:1786163 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 10:52:44,169 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:52:44,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:52:44,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:52:44,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:45,082 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:45,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:46,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:47,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:48,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:48,768 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:49,083 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml +2025-01-01 10:52:49,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:50,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:51,084 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:51,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:52,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:53,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:54,522 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:52:54,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:55,085 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:52:55,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:56,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:57,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:58,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:59,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:52:59,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:52:59,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:52:59,687 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:00,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:01,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:02,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:03,087 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:03,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:04,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:05,088 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:05,089 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:05,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:06,557 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:07,089 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:07,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:08,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:09,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:10,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:10,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:11,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:12,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:13,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:14,169 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:53:14,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:53:14,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:53:14,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:15,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:15,614 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:16,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:17,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:18,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:19,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:20,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:20,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:21,093 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:21,559 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:22,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:23,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:24,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:25,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:26,552 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:26,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:27,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:28,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:29,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:53:29,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:53:29,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:30,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:31,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:31,679 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:32,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:33,096 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:33,158 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:53:33,159 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 with policy now +2025-01-01 10:53:33,159 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:53:33,160 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:53:33,160 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:53:33,162 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:53:33,387 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/2mla3ytc-media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 +2025-01-01 10:53:33,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:34,096 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 +2025-01-01 10:53:34,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:53:34,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:53:34,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media +2025-01-01 10:53:34,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos +2025-01-01 10:53:34,401 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/dg1j6msb-media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 +2025-01-01 10:53:34,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:35,097 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:35,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:36,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:37,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:37,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:38,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:39,098 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:39,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:40,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:41,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:42,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:43,100 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:43,100 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:53:43,116 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:43,351 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/32fstrdh-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:53:43,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:44,169 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:53:44,548 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:53:44,548 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:53:44,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:45,100 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:45,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:46,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:47,101 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:53:47,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:48,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:49,071 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:49,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:50,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:51,564 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:52,564 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:53,564 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:54,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:53:54,564 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:55,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:56,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:57,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:58,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:59,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:53:59,550 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:53:59,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:53:59,610 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:00,572 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:01,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:02,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:03,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:04,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:05,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:05,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:06,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:07,568 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:08,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:09,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:10,553 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:10,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:11,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:12,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:13,569 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:14,170 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:54:14,288 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:54:14,288 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 with policy now +2025-01-01 10:54:14,289 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:54:14,290 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:54:14,290 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:54:14,290 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:54:14,514 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/zf34tc2c-media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 +2025-01-01 10:54:14,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:54:14,549 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:54:14,570 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:15,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:15,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:54:15,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 +2025-01-01 10:54:15,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:54:15,378 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/tao80fiw-media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 +2025-01-01 10:54:15,574 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:15,627 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:16,110 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:16,577 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:17,577 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:18,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:19,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:20,576 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:21,112 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:21,565 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:21,587 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:22,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:23,112 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:23,113 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:54:23,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:24,099 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:54:24,099 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:54:24,099 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:54:24,113 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:54:24,577 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:25,113 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:25,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:26,577 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:26,623 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:27,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:28,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:29,115 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:29,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:54:29,549 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:54:29,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:30,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:31,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:31,700 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:32,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:33,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:34,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:35,578 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:36,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:37,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:37,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:38,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:39,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:40,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:41,579 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:42,554 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:42,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:43,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:44,170 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:54:44,549 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:54:44,549 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:54:44,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:45,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:46,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:47,580 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:47,636 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:48,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:49,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:50,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:51,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:52,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:53,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:53,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:54,582 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:54,925 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:54:54,926 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:54:54,926 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 with policy now +2025-01-01 10:54:54,926 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:54:54,926 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:54:54,926 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:54:55,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:55,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:54:55,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 +2025-01-01 10:54:55,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:54:55,174 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/8aa7ymy1-media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 +2025-01-01 10:54:55,417 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/gtz2io06-media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 +2025-01-01 10:54:55,585 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:56,123 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:56,591 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:57,124 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:54:57,589 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:58,590 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:54:58,960 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:54:59,555 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:54:59,555 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:54:59,601 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:00,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:01,125 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:01,602 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:02,590 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:03,592 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:04,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:04,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:05,126 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:05,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:06,593 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:07,595 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:08,599 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:09,600 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:10,560 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:10,597 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:11,597 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:12,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:13,601 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:14,171 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:55:14,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:55:14,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:55:14,598 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:14,922 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:55:14,922 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:55:14,922 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:55:15,130 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:55:15,130 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:15,130 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:55:15,364 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/2zbmys5t-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:55:15,602 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:16,249 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:16,604 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:17,130 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:17,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:18,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:19,131 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:19,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:20,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:21,132 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:21,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:21,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:22,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:23,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:24,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:25,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:26,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:26,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:27,605 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:28,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:29,550 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:55:29,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:55:29,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:30,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:31,135 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:31,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:31,681 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:32,608 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:33,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:34,606 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:35,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:36,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:37,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:37,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:38,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:39,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:40,607 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:41,609 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:42,558 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:42,608 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:43,608 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:44,171 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:55:44,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:55:44,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:55:44,608 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:45,609 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:46,337 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:55:46,337 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 with policy now +2025-01-01 10:55:46,342 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:55:46,343 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:55:46,343 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:55:46,343 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:55:46,547 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/ozvfccbj-media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 +2025-01-01 10:55:46,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:47,141 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:47,141 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:55:47,141 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 +2025-01-01 10:55:47,141 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:55:47,403 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/q58ldh30-media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 +2025-01-01 10:55:47,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:48,375 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:48,613 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:49,614 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:50,616 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:51,615 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:52,630 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:53,143 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:53,615 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:53,635 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:54,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:54,970 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:55:54,970 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:55:54,970 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:55:55,143 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:55:55,144 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:55,144 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:55:55,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:56,623 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:57,144 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:55:57,618 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:58,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:55:58,669 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:55:59,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:55:59,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:55:59,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:00,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:01,146 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:01,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:02,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:03,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:04,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:04,619 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:05,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:06,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:07,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:08,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:09,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:09,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:10,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:11,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:12,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:13,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:14,171 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:56:14,551 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:56:14,551 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:56:14,620 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:14,679 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:15,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:16,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:17,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:18,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:19,152 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:19,621 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:20,561 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:20,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:21,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:22,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:23,622 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:24,623 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:25,562 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:25,623 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:26,630 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:26,990 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:56:26,990 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 with policy now +2025-01-01 10:56:26,995 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:56:26,996 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:56:26,996 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:26,996 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:27,154 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:27,154 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:27,155 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 +2025-01-01 10:56:27,155 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval +2025-01-01 10:56:27,210 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/w1vs6v9d-media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 +2025-01-01 10:56:27,508 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/yclzgvqr-media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 +2025-01-01 10:56:27,645 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:28,630 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:29,155 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:29,563 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:56:29,563 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:56:29,631 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:30,637 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:30,646 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:31,635 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:32,634 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:33,156 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:33,670 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:34,657 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:35,650 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:35,710 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:36,643 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:37,158 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:37,648 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:38,663 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:39,649 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:40,656 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:41,581 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:41,646 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:42,652 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:43,656 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:44,172 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:56:44,556 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 10:56:44,556 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: stop_status +2025-01-01 10:56:44,654 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:45,654 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:46,648 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:46,664 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:47,657 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:48,661 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:49,683 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:50,035 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:56:50,035 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:50,035 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:50,162 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:56:50,162 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:50,246 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 10:56:50,247 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 10:56:50,388 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:56:50,399 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/3qtsnd0z-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:56:50,659 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:50,671 INFO wandb-upload_2:1786163 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1r6scxd5 +2025-01-01 10:56:51,022 INFO SenderThread:1786163 [sender.py:send_request_log_artifact():1525] logged artifact run-qo464509-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY2Njg5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTIzOTEyMQ==', 'latestArtifact': None}} +2025-01-01 10:56:51,022 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:56:51,022 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 10:56:51,023 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:56:51,023 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:51,023 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:51,162 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table +2025-01-01 10:56:51,163 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval +2025-01-01 10:56:51,227 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/xj491i9b-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 10:56:51,488 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/r93183fk-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 10:56:51,660 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:52,024 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:56:52,660 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:53,428 DEBUG SenderThread:1786163 [sender.py:send():379] send: telemetry +2025-01-01 10:56:53,428 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,429 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,430 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,430 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,430 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,431 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,435 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,435 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,436 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,436 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,441 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,441 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,441 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,442 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,448 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,449 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,449 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,450 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,456 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,456 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,457 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:53,458 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:53,537 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 10:56:53,538 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 10:56:53,547 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 10:56:53,660 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:53,889 INFO wandb-upload_1:1786163 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwb2orbjw +2025-01-01 10:56:54,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:56:54,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:54,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json +2025-01-01 10:56:54,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval +2025-01-01 10:56:54,287 INFO SenderThread:1786163 [sender.py:send_request_log_artifact():1525] logged artifact run-qo464509-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NjkzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTIzOTEyNw==', 'latestArtifact': None}} +2025-01-01 10:56:54,287 DEBUG SenderThread:1786163 [sender.py:send():379] send: files +2025-01-01 10:56:54,287 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json with policy now +2025-01-01 10:56:54,288 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:56:54,289 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:56:54,290 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:54,461 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/et2enguc-media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json +2025-01-01 10:56:54,660 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:56:54,695 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/93m5r6pj-media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json +2025-01-01 10:56:55,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:56:55,164 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:55,361 DEBUG SenderThread:1786163 [sender.py:send():379] send: exit +2025-01-01 10:56:55,361 INFO SenderThread:1786163 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:send_exit():588] handling runtime: 311 +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:send_exit():594] send defer +2025-01-01 10:56:55,362 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,362 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 10:56:55,362 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 10:56:55,362 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 1 +2025-01-01 10:56:55,363 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,363 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 10:56:55,363 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:56:55,363 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 10:56:55,363 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 2 +2025-01-01 10:56:55,363 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,363 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 10:56:55,363 INFO HandlerThread:1786163 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 10:56:55,363 DEBUG SystemMonitor:1786163 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 10:56:55,363 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 10:56:55,364 DEBUG SystemMonitor:1786163 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 10:56:55,364 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 10:56:55,415 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 10:56:55,415 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 10:56:55,415 INFO HandlerThread:1786163 [interfaces.py:finish():200] Joined network monitor +2025-01-01 10:56:55,416 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:56:55,416 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 10:56:55,416 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 3 +2025-01-01 10:56:55,416 DEBUG SenderThread:1786163 [sender.py:send():379] send: stats +2025-01-01 10:56:55,416 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,416 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 10:56:55,416 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:56:55,417 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 10:56:55,417 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 4 +2025-01-01 10:56:55,417 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:56:55,417 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 10:56:56,165 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:56:57,165 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:57:02,044 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:02,044 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:57:02,045 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 10:57:02,046 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:02,046 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 10:57:02,046 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 5 +2025-01-01 10:57:02,046 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:02,046 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:02,046 DEBUG SenderThread:1786163 [sender.py:send():379] send: history +2025-01-01 10:57:02,046 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 10:57:02,046 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: summary_record +2025-01-01 10:57:02,047 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:57:02,048 DEBUG SenderThread:1786163 [sender.py:send():379] send: summary +2025-01-01 10:57:02,051 INFO SenderThread:1786163 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 10:57:02,051 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:02,052 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 10:57:02,052 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 6 +2025-01-01 10:57:02,052 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:02,052 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 10:57:02,052 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:02,052 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 10:57:02,055 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 10:57:02,166 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 7 +2025-01-01 10:57:02,166 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:02,166 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 10:57:02,167 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:02,167 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 10:57:02,167 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml +2025-01-01 10:57:02,167 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:57:02,362 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:04,074 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 8 +2025-01-01 10:57:04,074 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:04,074 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:04,075 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 10:57:04,075 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:04,075 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 10:57:04,075 INFO SenderThread:1786163 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 10:57:04,077 INFO SenderThread:1786163 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 10:57:04,079 INFO SenderThread:1786163 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 10:57:04,093 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 9 +2025-01-01 10:57:04,093 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:04,093 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 10:57:04,093 DEBUG SenderThread:1786163 [sender.py:send():379] send: artifact +2025-01-01 10:57:04,168 INFO Thread-12 :1786163 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:57:04,363 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:04,632 INFO wandb-upload_1:1786163 [upload_job.py:push():85] Skipped uploading /tmp/tmpqfdli0iz/wandb-job.json +2025-01-01 10:57:04,633 INFO wandb-upload_0:1786163 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmprokw9jhr +2025-01-01 10:57:04,768 INFO wandb-upload_2:1786163 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8m6d6qiy +2025-01-01 10:57:05,264 INFO SenderThread:1786163 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'versionIndex': 403}}} +2025-01-01 10:57:05,264 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:05,264 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 10:57:05,264 INFO SenderThread:1786163 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 10:57:06,169 INFO SenderThread:1786163 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files +2025-01-01 10:57:06,169 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml config.yaml +2025-01-01 10:57:06,169 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json wandb-summary.json +2025-01-01 10:57:06,169 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:57:06,171 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/requirements.txt requirements.txt +2025-01-01 10:57:06,171 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-metadata.json wandb-metadata.json +2025-01-01 10:57:06,171 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log output.log +2025-01-01 10:57:06,171 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/diff.patch diff.patch +2025-01-01 10:57:06,175 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 media/videos/eval/1_eval_1_561430e6869e3e143edf.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 media/videos/eval/3_eval_3_a5ed70decb9e07902125.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 media/videos/eval/4_eval_4_0f9d89dde9cf486c83fd.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 media/videos/eval/0_eval_0_047a5015a7283e01ecf8.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 media/videos/eval/2_eval_2_89284451a45c1f0bdd58.mp4 +2025-01-01 10:57:06,180 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json media/table/eval/total_success_rate_ci_6_1a02426d42619cd057a5.table.json +2025-01-01 10:57:06,181 INFO SenderThread:1786163 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 10:57:06,181 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 10 +2025-01-01 10:57:06,181 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:06,181 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,181 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 10:57:06,181 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,181 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 10:57:06,181 INFO SenderThread:1786163 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 10:57:06,344 INFO wandb-upload_1:1786163 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/config.yaml +2025-01-01 10:57:06,360 INFO wandb-upload_2:1786163 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/wandb-summary.json +2025-01-01 10:57:06,364 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:06,364 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:06,401 INFO wandb-upload_3:1786163 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/requirements.txt +2025-01-01 10:57:06,410 INFO wandb-upload_4:1786163 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/files/output.log +2025-01-01 10:57:06,443 INFO wandb-upload_0:1786163 [upload_job.py:push():130] Uploaded file /tmp/tmphoej7q7fwandb/uxvw1l74-events.out.tfevents.1735746704.leonmkim-ROG-Strix-G15CS-G15CS.1785977.0 +2025-01-01 10:57:06,644 INFO Thread-11 (_thread_body):1786163 [sender.py:transition_state():614] send defer: 11 +2025-01-01 10:57:06,644 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,644 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 10:57:06,644 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,644 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 10:57:06,644 INFO SenderThread:1786163 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 10:57:06,645 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 12 +2025-01-01 10:57:06,645 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,645 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 10:57:06,646 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,646 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 10:57:06,646 INFO SenderThread:1786163 [file_stream.py:finish():601] file stream finish called +2025-01-01 10:57:06,839 INFO SenderThread:1786163 [file_stream.py:finish():605] file stream finish is done +2025-01-01 10:57:06,839 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 13 +2025-01-01 10:57:06,839 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,839 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 10:57:06,839 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,839 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 10:57:06,839 INFO SenderThread:1786163 [sender.py:transition_state():614] send defer: 14 +2025-01-01 10:57:06,840 DEBUG SenderThread:1786163 [sender.py:send():379] send: final +2025-01-01 10:57:06,840 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: defer +2025-01-01 10:57:06,840 DEBUG SenderThread:1786163 [sender.py:send():379] send: footer +2025-01-01 10:57:06,840 INFO HandlerThread:1786163 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 10:57:06,840 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: defer +2025-01-01 10:57:06,840 INFO SenderThread:1786163 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 10:57:06,840 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:06,841 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 10:57:06,841 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 10:57:06,841 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:06,842 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: server_info +2025-01-01 10:57:06,843 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 10:57:06,844 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 10:57:06,845 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 10:57:06,896 DEBUG SenderThread:1786163 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 10:57:06,896 INFO MainThread:1786163 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 10:57:06,897 INFO MainThread:1786163 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 10:57:06,897 INFO MainThread:1786163 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 10:57:06,897 DEBUG HandlerThread:1786163 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 10:57:06,897 INFO HandlerThread:1786163 [handler.py:finish():882] shutting down handler +2025-01-01 10:57:07,841 INFO WriterThread:1786163 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/run-qo464509.wandb +2025-01-01 10:57:07,896 INFO SenderThread:1786163 [sender.py:finish():1615] shutting down sender +2025-01-01 10:57:07,897 INFO SenderThread:1786163 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 10:57:07,897 INFO SenderThread:1786163 [file_pusher.py:join():175] waiting for file pusher diff --git a/qo464509/wandb/run-20250101_105143-qo464509/logs/debug.log b/qo464509/wandb/run-20250101_105143-qo464509/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..77cb70a24ba6ece91528a05a44cd1ba863e540e1 --- /dev/null +++ b/qo464509/wandb/run-20250101_105143-qo464509/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Configure stats pid to 1785977 +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/settings +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/logs/debug.log +2025-01-01 10:51:43,759 INFO MainThread:1785977 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/wandb/run-20250101_105143-qo464509/logs/debug-internal.log +2025-01-01 10:51:43,760 INFO MainThread:1785977 [wandb_init.py:init():569] calling init triggers +2025-01-01 10:51:43,760 INFO MainThread:1785977 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509'} +2025-01-01 10:51:43,760 INFO MainThread:1785977 [wandb_init.py:init():619] starting backend +2025-01-01 10:51:43,760 INFO MainThread:1785977 [wandb_init.py:init():623] setting up manager +2025-01-01 10:51:43,765 INFO MainThread:1785977 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 10:51:43,767 INFO MainThread:1785977 [wandb_init.py:init():631] backend started and connected +2025-01-01 10:51:43,779 INFO MainThread:1785977 [wandb_init.py:init():720] updated telemetry +2025-01-01 10:51:43,785 INFO MainThread:1785977 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 10:51:44,083 INFO MainThread:1785977 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 10:51:44,154 INFO MainThread:1785977 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 10:51:44,154 INFO MainThread:1785977 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 10:51:44,547 INFO MainThread:1785977 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 10:51:44,547 INFO MainThread:1785977 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 10:51:44,548 INFO MainThread:1785977 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 10:51:44,548 INFO MainThread:1785977 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 10:51:44,550 INFO MainThread:1785977 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 10:51:44,551 INFO MainThread:1785977 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/qo464509/tb, True +2025-01-01 10:52:34,330 INFO MainThread:1785977 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-01 10:57:07,898 WARNING MsgRouterThr:1785977 [router.py:message_loop():77] message_loop has been closed diff --git a/qo464509/wandb/run-20250101_105143-qo464509/run-qo464509.wandb b/qo464509/wandb/run-20250101_105143-qo464509/run-qo464509.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2b24025b60a923b1fc059b5be7cd9e5dbba55791 Binary files /dev/null and b/qo464509/wandb/run-20250101_105143-qo464509/run-qo464509.wandb differ diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..fde4fa91324a9bbdb4ce51b44f8ec7b3f1fefe5d --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67ff6067c41537bf9c13127c53810bd9d3f3b71585597823c0824d85da6afb34 +size 910102870 diff --git a/snapshot_11249.pt b/snapshot_11249.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4ac03bd9c1c74bdf24af46be3ae9f37b6890441 --- /dev/null +++ b/snapshot_11249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06ecd654c7a2bd3345ad019e013e32e7ad4d9e81c78453b8797477b7ce16f261 +size 910102870 diff --git a/snapshot_11499.pt b/snapshot_11499.pt new file mode 100644 index 0000000000000000000000000000000000000000..754bdc5422ae55f64460d6d2e72e48db21a5d2e5 --- /dev/null +++ b/snapshot_11499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47db6890d7c6b0f25dd843e59b2fbef2cf7829534119f5d00157d5fc39986bae +size 910102870 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..9044b33c64eda883b82b876818d0f9588a5e4e2b --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddebf87fffadfe96663b99396fca5098ac2381e051ee13ab28e3ee282c4ec4f0 +size 910102870 diff --git a/snapshot_14499.pt b/snapshot_14499.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a95adf7bbaf89aaef324b490ad64c108676b1e2 --- /dev/null +++ b/snapshot_14499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84d69389dbad2e2b9db875436c7923f426a30b1f52545eb1fb51d7e33ea3b667 +size 910102870 diff --git a/snapshot_14999.pt b/snapshot_14999.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b012c780425700725ca5985132675541cdb0a5c --- /dev/null +++ b/snapshot_14999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60dfcbd2fc4dbf2890532080ee1c1ee296f670b3d0bdf9b205e7a4633d9503ac +size 910102870 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3ecd9b660370ae1af515d5c3bc95e6ae38a2d77 --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fee7421dba07b5f26c621257a115fb079dff443c0f51ed947f454026f7fea24 +size 910102360 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..faf779fdd62bc7ff9344504b671db853b0c9a6d6 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0f0d64453c22acc7de000ffff17409cfc39fb4f925136a107ae99b5ab3abbac +size 910102870 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..18eb647dc8a9432637903049cb0f1d1f31718b19 --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a72fdaf188accca99aefea854feb8d7d1011fdbbaa4f9c63ccc5a1817bc57b4 +size 910102360 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..579d7659b971c838bbab252b6c202582e3a24807 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5a904b3d230fa81f7d2b3d4ddb8d500bfae4e358683e3451bf9ea51a53a458e +size 910102360 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c18747d6c0debcf0806923134709597f8023d1f --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cef1729f4f22a042ce71ee691077ad916e768b11984970160191c9f7756638ba +size 910102870 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea61f8ee1b47d50337d3f92f195863d44b609c79 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4afbb88992e87f02cdae3fff9cdc00101a4e54af97a28205e92aa7af94d20764 +size 910102870 diff --git a/snapshot_8249.pt b/snapshot_8249.pt new file mode 100644 index 0000000000000000000000000000000000000000..1228a35aa68f5b56c373b5416c28ccc5afd620ac --- /dev/null +++ b/snapshot_8249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3037da86d9b21d5ade3fa3cdff63f0a572134923a2ed36e4ca6be7b9c746ea82 +size 910102870 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2b3110cc4339c7273f2309969c6fbfb79cd07d9 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c68607e84186b201cec0adb2a8966588b6f5e9be8c8ce943a6d69b28d828146 +size 910102870 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..faf779fdd62bc7ff9344504b671db853b0c9a6d6 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0f0d64453c22acc7de000ffff17409cfc39fb4f925136a107ae99b5ab3abbac +size 910102870