diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..0af49d89eb68d24804860cbb8c7bf4fa68396320 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,33 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/2uvisx3r/.hydra/config.yaml b/2uvisx3r/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/2uvisx3r/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/2uvisx3r/.hydra/hydra.yaml b/2uvisx3r/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5deed3a8cfeb6cfa68baa7bf404ad0fd667bead --- /dev/null +++ b/2uvisx3r/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/2uvisx3r/.hydra/overrides.yaml b/2uvisx3r/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/2uvisx3r/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/2uvisx3r/episode_rosbags/aligned_depth_to_color_K.npy b/2uvisx3r/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/2uvisx3r/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/2uvisx3r/episode_rosbags/cam_tf_world.npy b/2uvisx3r/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/2uvisx3r/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/2uvisx3r/episode_rosbags/color_K.npy b/2uvisx3r/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/2uvisx3r/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/2uvisx3r/episode_rosbags/depth_K.npy b/2uvisx3r/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/2uvisx3r/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/2uvisx3r/eval_robot.log b/2uvisx3r/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..3942b065a9737dfec1c56ec674453c3c36e80b25 --- /dev/null +++ b/2uvisx3r/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 20:34:17,276][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:34:17,279][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/2uvisx3r/eval_video/0_eval.mp4 b/2uvisx3r/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f1b5d1b7d0f2b520aeb6d93185c33e06db05ecbf --- /dev/null +++ b/2uvisx3r/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:733d8003db33e86d4c9306deca409a10ad441137d5f571fd33956c260dcfaf23 +size 1133946 diff --git a/2uvisx3r/eval_video/1_eval.mp4 b/2uvisx3r/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..093e5b8d8ae5f6ca75d649c3d6e3613710f73492 --- /dev/null +++ b/2uvisx3r/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59fdd9b7330343a76e3c3136f94d1b903fb3f1bcbcd74a2013c23bb9714174be +size 1209518 diff --git a/2uvisx3r/eval_video/2_eval.mp4 b/2uvisx3r/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6a71047ecb0415ddfaafa7a531f10bf7e2bc967 --- /dev/null +++ b/2uvisx3r/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d54e64f7a05a499157babbc41d91945357bf22b215de5365e129c0888d9a842 +size 1072150 diff --git a/2uvisx3r/eval_video/3_eval.mp4 b/2uvisx3r/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab02b3dd9fc54089aecc50739d1d6f1eafb94ab5 --- /dev/null +++ b/2uvisx3r/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f279f9c9369c3b87c444ccadd04c31defe965ead14f924a256b420181a8673da +size 1146457 diff --git a/2uvisx3r/eval_video/4_eval.mp4 b/2uvisx3r/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b8f1229f65d5a1cfc922574c1410078758667e2 --- /dev/null +++ b/2uvisx3r/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a8eddcd55e59db846f73f4cacee5bd18cc056c973203e17c607c793ed2a372e +size 1159741 diff --git a/2uvisx3r/tb/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 b/2uvisx3r/tb/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 new file mode 100644 index 0000000000000000000000000000000000000000..2564ff81541fcd2139e75c87028637210d40229a --- /dev/null +++ b/2uvisx3r/tb/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f52603fcbceddfc400a5bda0f8b85c601a3f5400ccd86cae2b08fde9a3f4bfcc +size 6539 diff --git a/2uvisx3r/wandb/debug-internal.log b/2uvisx3r/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..902749567ea6335abd11b3e88328762ec17ac91b --- /dev/null +++ b/2uvisx3r/wandb/debug-internal.log @@ -0,0 +1,798 @@ +2025-01-07 20:34:23,021 INFO StreamThr :2228203 [internal.py:wandb_internal():85] W&B internal server running at pid: 2228203, started at: 2025-01-07 20:34:23.020938 +2025-01-07 20:34:23,022 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:34:23,033 INFO WriterThread:2228203 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/run-2uvisx3r.wandb +2025-01-07 20:34:23,033 DEBUG SenderThread:2228203 [sender.py:send():379] send: header +2025-01-07 20:34:23,043 DEBUG SenderThread:2228203 [sender.py:send():379] send: run +2025-01-07 20:34:23,336 INFO SenderThread:2228203 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files +2025-01-07 20:34:23,336 INFO SenderThread:2228203 [sender.py:_start_run_threads():1188] run started: 2uvisx3r with start time 1736300063.020979 +2025-01-07 20:34:23,350 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:34:23,350 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:34:23,423 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:34:23,428 DEBUG HandlerThread:2228203 [system_info.py:__init__():26] System info init +2025-01-07 20:34:23,428 DEBUG HandlerThread:2228203 [system_info.py:__init__():41] System info init done +2025-01-07 20:34:23,428 INFO HandlerThread:2228203 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:34:23,428 INFO SystemMonitor:2228203 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:34:23,428 INFO HandlerThread:2228203 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:34:23,429 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:34:23,429 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:34:23,430 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:34:23,430 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:34:23,431 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:34:23,442 DEBUG HandlerThread:2228203 [system_info.py:probe():152] Probing system +2025-01-07 20:34:23,443 DEBUG HandlerThread:2228203 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:34:23,448 DEBUG HandlerThread:2228203 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:34:23,448 DEBUG HandlerThread:2228203 [system_info.py:probe():200] Probing system done +2025-01-07 20:34:23,448 DEBUG HandlerThread:2228203 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:34:23.442284', 'startedAt': '2025-01-08T01:34:23.013073', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.1075, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.041, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 593.6038970947266}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:34:23,448 INFO HandlerThread:2228203 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:34:23,448 INFO HandlerThread:2228203 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:34:23,448 DEBUG HandlerThread:2228203 [system_info.py:_save_code():44] Saving code +2025-01-07 20:34:23,449 DEBUG HandlerThread:2228203 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:34:23,449 DEBUG HandlerThread:2228203 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:34:23,511 DEBUG HandlerThread:2228203 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:34:23,512 INFO HandlerThread:2228203 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:34:23,515 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:34:23,515 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:34:23,515 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:34:23,515 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:34:23,756 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:34:23,756 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:34:23,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:34:23,757 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:34:23,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:23,759 INFO HandlerThread:2228203 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/tb" + save: true +} +control { + relay_id: "a56d13a1fbde40a9bc9a1c91f550caa5" +} +_info { + stream_id: "2uvisx3r" +} + +2025-01-07 20:34:23,759 DEBUG HandlerThread:2228203 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:34:23,767 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/5u6zzksn-diff.patch +2025-01-07 20:34:23,778 INFO wandb-upload_0:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/02fvii0r-wandb-metadata.json +2025-01-07 20:34:23,817 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/rv78ku7a-code/FISH/eval_robot.py +2025-01-07 20:34:23,893 DEBUG SenderThread:2228203 [sender.py:send():379] send: telemetry +2025-01-07 20:34:23,893 DEBUG SenderThread:2228203 [sender.py:send():379] send: tbrecord +2025-01-07 20:34:23,894 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:34:23,894 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 with policy live +2025-01-07 20:34:24,084 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/yebrvrg8-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/requirements.txt +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code/FISH/eval_robot.py +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-metadata.json +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/diff.patch +2025-01-07 20:34:24,339 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:24,339 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code +2025-01-07 20:34:24,339 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code/FISH +2025-01-07 20:34:24,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:25,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:26,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:26,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:27,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:28,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:28,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:29,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:30,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:31,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:32,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:33,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:33,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:34,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:35,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:36,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:37,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:38,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:34:38,757 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:34:38,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:38,894 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:39,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:40,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:41,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:42,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:43,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:44,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:44,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:45,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:46,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:47,179 DEBUG SenderThread:2228203 [sender.py:send():379] send: config +2025-01-07 20:34:47,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:48,344 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:48,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:49,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:49,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:50,344 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:50,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:51,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:52,345 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:52,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:53,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:34:53,758 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:34:53,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:54,345 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:54,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:55,662 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:55,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:56,346 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:56,346 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml +2025-01-07 20:34:56,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:57,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:58,347 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:58,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:59,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:00,348 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:00,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:01,679 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:01,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:02,348 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:02,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:03,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:04,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:05,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:06,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:06,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:07,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:08,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:35:08,757 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:35:08,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:09,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:10,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:11,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:11,835 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:12,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:13,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:14,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:15,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:16,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:17,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:17,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:18,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:19,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:20,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:21,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:22,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:22,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:23,431 DEBUG SystemMonitor:2228203 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:35:23,432 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:35:23,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:35:23,758 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:35:23,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:24,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:25,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:26,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:27,328 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:35:27,328 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 with policy now +2025-01-07 20:35:27,329 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:35:27,329 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:35:27,330 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:35:27,330 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:35:27,612 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/yrn5mbvj-media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 +2025-01-07 20:35:27,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:27,901 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/gnanrqee-media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 +2025-01-07 20:35:28,334 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:28,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:28,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:29,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:30,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:31,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:32,356 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:32,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:33,649 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:33,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:34,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:35,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:36,358 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:36,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:37,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:38,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:38,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:35:38,763 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:35:38,805 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:39,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:40,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:41,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:42,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:43,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:43,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:44,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:45,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:46,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:47,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:48,361 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:35:48,626 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/vav8km2y-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:35:48,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:49,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:49,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:50,362 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:50,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:51,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:52,363 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:52,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:53,432 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:35:53,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:35:53,761 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:35:53,809 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:54,364 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:54,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:54,908 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:55,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:56,786 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:57,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:58,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:59,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:00,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:00,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:01,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:02,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:03,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:04,783 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:05,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:05,782 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:06,782 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:07,782 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:08,371 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:08,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:36:08,761 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:36:08,805 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:09,783 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:10,783 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:10,920 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:11,798 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:12,798 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:13,798 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:14,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:15,802 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:16,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:16,809 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:17,810 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:18,810 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:19,810 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:20,814 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:20,945 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:36:20,954 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:36:20,955 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 with policy now +2025-01-07 20:36:20,955 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:36:20,955 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:36:20,955 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:36:21,222 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/pcq6w7qv-media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 +2025-01-07 20:36:21,375 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:36:21,375 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 +2025-01-07 20:36:21,376 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:36:21,647 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/8fj8v00n-media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 +2025-01-07 20:36:21,818 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:21,956 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:22,376 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:22,814 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:23,433 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:36:23,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:36:23,769 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:36:23,815 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:24,816 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:25,823 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:26,377 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:26,823 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:27,183 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:27,819 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:28,378 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:28,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:29,825 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:30,823 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:31,823 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:32,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:32,822 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:33,822 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:34,825 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:35,825 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:36,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:37,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:37,979 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:38,381 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:36:38,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:36:38,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:36:38,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:39,195 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:36:39,195 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:36:39,196 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:36:39,381 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:36:39,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:40,381 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:40,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:41,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:42,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:43,192 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:43,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:44,383 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:44,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:45,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:46,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:47,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:48,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:48,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:49,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:50,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:51,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:52,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:53,433 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:36:53,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:36:53,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:36:53,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:53,901 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:54,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:55,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:56,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:57,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:58,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:59,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:59,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:00,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:01,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:02,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:03,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:04,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:05,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:05,830 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:06,830 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:07,830 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:08,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:37:08,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:37:08,830 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:09,833 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:09,949 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:37:09,950 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:37:09,950 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 with policy now +2025-01-07 20:37:09,950 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:37:09,950 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:37:09,951 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:37:10,231 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/bf8wuoru-media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 +2025-01-07 20:37:10,391 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:37:10,391 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:10,391 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 +2025-01-07 20:37:10,392 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:37:10,652 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/sqnbzomx-media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 +2025-01-07 20:37:10,833 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:10,954 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:11,835 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:12,392 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:12,836 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:13,837 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:14,392 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:14,838 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:15,841 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:16,393 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:16,455 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:16,835 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:17,839 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:18,394 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:18,839 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:19,841 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:20,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:21,771 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:21,838 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:22,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:23,030 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:37:23,030 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:37:23,031 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:37:23,396 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:37:23,396 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:37:23,434 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:37:23,642 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/n0cvzznu-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:37:23,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:37:23,766 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:37:23,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:24,396 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:24,845 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:25,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:26,848 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:27,345 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:27,839 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:28,397 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:28,881 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:29,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:30,398 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:30,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:31,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:32,399 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:32,529 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:32,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:33,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:34,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:35,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:36,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:37,690 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:37,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:38,401 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:38,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:37:38,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:37:38,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:39,842 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:40,401 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:40,842 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:41,842 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:42,402 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:42,721 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:42,842 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:43,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:44,403 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:44,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:45,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:46,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:47,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:47,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:48,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:49,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:50,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:51,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:52,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:52,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:53,434 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:37:53,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:37:53,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:37:53,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:54,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:55,847 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:56,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:56,657 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:37:56,657 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 with policy now +2025-01-07 20:37:56,664 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:37:56,665 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:37:56,665 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:37:56,665 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:37:56,847 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:56,913 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/1oh8qdpv-media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 +2025-01-07 20:37:57,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:37:57,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 +2025-01-07 20:37:57,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:37:57,687 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/smlnlylj-media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 +2025-01-07 20:37:57,852 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:58,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:58,707 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:58,851 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:59,849 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:00,861 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:01,861 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:02,409 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:02,872 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:03,777 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:03,856 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:04,410 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:04,854 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:05,861 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:06,861 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:07,862 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:08,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:38:08,770 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:38:08,868 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:08,888 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:09,867 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:10,412 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:38:10,862 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:11,089 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:11,089 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:11,089 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:11,412 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:11,862 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:12,412 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:12,866 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:13,876 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:14,092 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:14,862 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:15,863 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:16,414 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:16,863 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:17,864 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:18,863 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:19,771 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:19,867 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:20,884 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:21,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:22,877 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:23,435 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:38:23,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:38:23,769 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:38:23,877 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:24,877 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:24,921 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:25,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:26,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:27,879 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:28,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:29,879 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:30,772 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:30,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:31,880 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:32,419 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:32,881 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:33,880 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:34,880 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:35,772 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:35,880 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:36,881 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:37,887 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:38,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:38:38,766 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:38:38,885 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:39,886 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:40,875 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:40,887 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:41,886 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:42,913 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:43,895 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:43,956 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:38:43,956 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 with policy now +2025-01-07 20:38:43,960 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:38:43,961 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:43,961 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:43,962 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:44,218 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/ri231522-media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 +2025-01-07 20:38:44,423 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:44,423 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:44,423 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 +2025-01-07 20:38:44,424 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:38:44,730 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/73bz6bw5-media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 +2025-01-07 20:38:44,899 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:45,903 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:45,978 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:46,424 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:46,906 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:47,906 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:48,424 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:48,907 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:49,914 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:50,911 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:51,473 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:51,911 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:52,426 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:52,906 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:53,435 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:38:53,773 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:38:53,773 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:38:53,917 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:54,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:55,138 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:55,139 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:55,140 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:55,427 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:55,427 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:38:55,674 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/rpdg1y7x-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:38:55,913 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:56,027 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:38:56,027 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:38:56,193 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:38:56,427 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:56,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:38:56,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media +2025-01-07 20:38:56,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval +2025-01-07 20:38:56,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table +2025-01-07 20:38:56,510 INFO wandb-upload_2:2228203 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp5svb0nm +2025-01-07 20:38:56,633 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/h9t81edn-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:38:56,684 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:38:56,693 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:38:56,901 INFO SenderThread:2228203 [sender.py:send_request_log_artifact():1525] logged artifact run-2uvisx3r-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQwNDIyMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzNjEyOA==', 'latestArtifact': None}} +2025-01-07 20:38:56,901 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:38:56,901 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:38:56,901 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:56,901 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,901 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:56,902 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,902 DEBUG SenderThread:2228203 [sender.py:send():379] send: telemetry +2025-01-07 20:38:56,902 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,903 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,903 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,903 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,905 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,905 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,905 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,906 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,906 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,906 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,906 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,907 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,907 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,907 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,907 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,907 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,908 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,908 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,908 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,908 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,909 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,909 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,909 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:38:56,912 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:57,152 INFO wandb-upload_0:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/3v8spda0-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:38:57,320 INFO wandb-upload_1:2228203 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprply37rq +2025-01-07 20:38:57,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:57,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:38:57,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json +2025-01-07 20:38:57,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval +2025-01-07 20:38:57,706 INFO SenderThread:2228203 [sender.py:send_request_log_artifact():1525] logged artifact run-2uvisx3r-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQwNDI0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzNjEzMQ==', 'latestArtifact': None}} +2025-01-07 20:38:57,707 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:38:57,707 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json with policy now +2025-01-07 20:38:57,707 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:57,707 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:57,708 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:57,883 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/pa6cqtzh-media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json +2025-01-07 20:38:57,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:58,131 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/vdn7kbpn-media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json +2025-01-07 20:38:58,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:58,429 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:58,913 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:59,913 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:00,914 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:01,914 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:02,773 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:02,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:03,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:04,914 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:05,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:06,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:07,773 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:07,917 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:08,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:39:08,766 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:39:08,916 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:09,553 DEBUG SenderThread:2228203 [sender.py:send():379] send: exit +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:send_exit():588] handling runtime: 286 +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:send_exit():594] send defer +2025-01-07 20:39:09,554 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,554 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:39:09,554 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:39:09,555 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:39:09,555 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,555 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:39:09,555 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:09,555 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:39:09,555 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:39:09,555 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,555 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:39:09,555 INFO HandlerThread:2228203 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:39:09,555 DEBUG SystemMonitor:2228203 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:39:09,555 DEBUG SystemMonitor:2228203 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:39:09,556 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:39:09,556 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:39:09,611 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:39:09,612 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:39:09,612 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:39:09,612 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:09,612 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:39:09,612 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:39:09,612 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:39:09,613 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,613 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:39:09,613 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:09,613 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:39:09,613 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:39:09,613 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,613 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:39:10,433 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:39:10,433 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:39:14,435 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml +2025-01-07 20:39:16,149 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:16,149 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:16,149 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:39:16,149 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,150 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:39:16,150 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:39:16,150 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:16,150 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,151 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:39:16,151 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:39:16,151 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:39:16,152 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:39:16,152 DEBUG SenderThread:2228203 [sender.py:send():379] send: summary +2025-01-07 20:39:16,155 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:39:16,156 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,156 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:39:16,156 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:39:16,156 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,156 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:39:16,156 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,156 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:39:16,156 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:39:16,156 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:16,156 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,156 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:39:16,157 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,157 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:39:16,235 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:39:16,235 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,235 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:39:16,235 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,235 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:39:16,235 INFO SenderThread:2228203 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:39:16,238 INFO SenderThread:2228203 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:39:16,240 INFO SenderThread:2228203 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:39:16,249 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:39:16,249 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,249 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:39:16,250 DEBUG SenderThread:2228203 [sender.py:send():379] send: artifact +2025-01-07 20:39:16,436 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:39:16,436 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:39:16,449 INFO SenderThread:2228203 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:39:16,449 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,450 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:39:16,450 INFO SenderThread:2228203 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:39:16,555 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:17,436 INFO SenderThread:2228203 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml config.yaml +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json wandb-summary.json +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/requirements.txt requirements.txt +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log output.log +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/diff.patch diff.patch +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:39:17,439 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 +2025-01-07 20:39:17,439 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 +2025-01-07 20:39:17,440 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 +2025-01-07 20:39:17,440 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 +2025-01-07 20:39:17,440 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 +2025-01-07 20:39:17,444 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json +2025-01-07 20:39:17,444 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:39:17,444 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:39:17,444 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:17,444 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:17,444 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:39:17,446 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:17,446 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:39:17,446 INFO SenderThread:2228203 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:39:17,555 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:17,555 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:17,636 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml +2025-01-07 20:39:17,655 INFO wandb-upload_0:2228203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/requirements.txt +2025-01-07 20:39:17,670 INFO wandb-upload_3:2228203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:39:17,672 INFO wandb-upload_4:2228203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:39:17,691 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/q2lcdjtn-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:39:17,892 INFO Thread-11 (_thread_body):2228203 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:39:17,892 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:17,892 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:39:17,892 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:17,892 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:39:17,892 INFO SenderThread:2228203 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:39:17,893 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:39:17,894 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:17,894 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:39:17,894 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:17,894 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:39:17,894 INFO SenderThread:2228203 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:39:18,115 INFO SenderThread:2228203 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:39:18,115 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:39:18,116 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:18,116 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:39:18,116 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:18,116 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:39:18,116 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:39:18,116 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:18,116 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:39:18,116 DEBUG SenderThread:2228203 [sender.py:send():379] send: final +2025-01-07 20:39:18,116 DEBUG SenderThread:2228203 [sender.py:send():379] send: footer +2025-01-07 20:39:18,116 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:18,116 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:39:18,117 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:18,117 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:39:18,117 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:18,117 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:39:18,118 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:39:18,118 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:18,118 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:39:18,118 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:18,172 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:18,172 INFO MainThread:2228203 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:39:18,172 INFO MainThread:2228203 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:39:18,173 INFO MainThread:2228203 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:39:18,173 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:39:18,173 INFO HandlerThread:2228203 [handler.py:finish():882] shutting down handler +2025-01-07 20:39:19,118 INFO WriterThread:2228203 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/run-2uvisx3r.wandb +2025-01-07 20:39:19,172 INFO SenderThread:2228203 [sender.py:finish():1615] shutting down sender +2025-01-07 20:39:19,172 INFO SenderThread:2228203 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:39:19,172 INFO SenderThread:2228203 [file_pusher.py:join():175] waiting for file pusher diff --git a/2uvisx3r/wandb/debug.log b/2uvisx3r/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..13271972d84400633b193c758bf38e23f05e0143 --- /dev/null +++ b/2uvisx3r/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Configure stats pid to 2228014 +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/settings +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug.log +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug-internal.log +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r'} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:init():619] starting backend +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:init():623] setting up manager +2025-01-07 20:34:23,020 INFO MainThread:2228014 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:34:23,020 INFO MainThread:2228014 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:34:23,032 INFO MainThread:2228014 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:34:23,040 INFO MainThread:2228014 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:34:23,349 INFO MainThread:2228014 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:34:23,418 INFO MainThread:2228014 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:34:23,418 INFO MainThread:2228014 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:34:23,757 INFO MainThread:2228014 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:34:23,757 INFO MainThread:2228014 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:34:23,757 INFO MainThread:2228014 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:34:23,757 INFO MainThread:2228014 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:34:23,758 INFO MainThread:2228014 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:34:23,759 INFO MainThread:2228014 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/tb, True +2025-01-07 20:34:47,179 INFO MainThread:2228014 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:39:19,174 WARNING MsgRouterThr:2228014 [router.py:message_loop():77] message_loop has been closed diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code/FISH/eval_robot.py b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b7df90f337e90f3a3b1056cb03ba2bf2544a7d1 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736300063 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/diff.patch b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a994978ad1dd8e2d18bdda54a765e73f2c0b5be --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/diff.patch @@ -0,0 +1,19 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..bc04fba 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -135,12 +135,12 @@ load_checkpoint: ${agent.load_checkpoint} + # RGBD+mask+act history + # wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 +-# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d911555e45f52099d5690a36b7348eac11aeb804 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 55, 0.45454545454545453, 0.30244287852423735, 0.6118613779321108, 0.33041951808744774, 0.5936293899825711, 0.34962770345834293, 0.5627395361536432, 0.36513858291223766, 0.5477405264878159]]} \ No newline at end of file diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f1b5d1b7d0f2b520aeb6d93185c33e06db05ecbf --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:733d8003db33e86d4c9306deca409a10ad441137d5f571fd33956c260dcfaf23 +size 1133946 diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..093e5b8d8ae5f6ca75d649c3d6e3613710f73492 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59fdd9b7330343a76e3c3136f94d1b903fb3f1bcbcd74a2013c23bb9714174be +size 1209518 diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6a71047ecb0415ddfaafa7a531f10bf7e2bc967 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d54e64f7a05a499157babbc41d91945357bf22b215de5365e129c0888d9a842 +size 1072150 diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab02b3dd9fc54089aecc50739d1d6f1eafb94ab5 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f279f9c9369c3b87c444ccadd04c31defe965ead14f924a256b420181a8673da +size 1146457 diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b8f1229f65d5a1cfc922574c1410078758667e2 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a8eddcd55e59db846f73f4cacee5bd18cc056c973203e17c607c793ed2a372e +size 1159741 diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f2d68bef75ddaaa81d81323513990a6c43c95e40 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log @@ -0,0 +1,267 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736300087.179752]: resetting environment +[INFO] [1736300087.185182]: cleared current plan +[INFO] [1736300087.186701]: moving to home +[INFO] [1736300088.189315]: reached home +[INFO] [1736300088.191263]: reset action history +[INFO] [1736300090.897764]: environment reset +Starting episode 0 +[INFO] [1736300090.900032]: resetting environment +[INFO] [1736300090.903069]: cleared current plan +[INFO] [1736300090.904033]: moving to home +[INFO] [1736300091.906200]: reached home +[INFO] [1736300091.907663]: reset action history +[INFO] [1736300094.614503]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736300095.675170]: resetting environment +[INFO] [1736300095.677779]: cleared current plan +[INFO] [1736300095.678781]: moving to home +[INFO] [1736300096.681296]: reached home +[INFO] [1736300096.684171]: reset action history +[INFO] [1736300099.391816]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/episode_rosbags/episode_0_2025-01-07-20-34-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2228014.log +started roslaunch server http://158.130.50.37:37955/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2228306] +process[print_text-2]: started with pid [2228330] +process[rosbag_record-3]: started with pid [2228331] +started bagging! +[INFO] [1736300126.391519]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736300127.131642]: Stored episode 1. +[INFO] [1736300127.131915]: Saving video... +[INFO] [1736300127.317923]: Video saved! +Starting episode 1 +[INFO] [1736300127.329312]: resetting environment +[INFO] [1736300127.333057]: cleared current plan +[INFO] [1736300127.333391]: moving to home +[INFO] [1736300130.940542]: reached home +[INFO] [1736300130.941011]: reset action history +[INFO] [1736300133.649182]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300147.576527]: resetting environment +[INFO] [1736300147.587495]: cleared current plan +[INFO] [1736300147.588120]: moving to home +[INFO] [1736300148.589269]: reached home +[INFO] [1736300148.589715]: reset action history +[INFO] [1736300151.296553]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/episode_rosbags/episode_1_2025-01-07-20-35-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2228014.log +started roslaunch server http://158.130.50.37:41779/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2228467] +process[print_text-5]: started with pid [2228468] +process[rosbag_record-6]: started with pid [2228469] +started bagging! +For topic gripper_width: timestamp difference is 100992362 for nearest: 1736300166269328753 - target: 1736300166168336391 at idx 184 +[INFO] [1736300179.909604]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736300180.680769]: Stored episode 2. +[INFO] [1736300180.683143]: Saving video... +[INFO] [1736300180.904084]: Video saved! +Starting episode 2 +[INFO] [1736300180.943041]: resetting environment +[INFO] [1736300180.954515]: cleared current plan +[INFO] [1736300180.954789]: moving to home +[INFO] [1736300184.458275]: reached home +[INFO] [1736300184.463190]: reset action history +[INFO] [1736300187.182867]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300197.994622]: resetting environment +[INFO] [1736300198.009203]: cleared current plan +[INFO] [1736300198.012645]: moving to home +[INFO] [1736300199.017082]: reached home +[INFO] [1736300199.020152]: reset action history +[INFO] [1736300201.739230]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/episode_rosbags/episode_2_2025-01-07-20-36-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2228014.log +started roslaunch server http://158.130.50.37:35755/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2228604] +process[print_text-8]: started with pid [2228605] +process[rosbag_record-9]: started with pid [2228606] +started bagging! +[INFO] [1736300228.893947]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736300229.677949]: Stored episode 3. +[INFO] [1736300229.678749]: Saving video... +[INFO] [1736300229.906273]: Video saved! +Starting episode 3 +[INFO] [1736300229.936531]: resetting environment +[INFO] [1736300229.947927]: cleared current plan +[INFO] [1736300229.949449]: moving to home +[INFO] [1736300233.571896]: reached home +[INFO] [1736300233.575424]: reset action history +[INFO] [1736300236.454184]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300242.529485]: resetting environment +[INFO] [1736300242.552096]: cleared current plan +[INFO] [1736300242.559299]: moving to home +[INFO] [1736300243.561496]: reached home +[INFO] [1736300243.564773]: reset action history +[INFO] [1736300246.301654]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/episode_rosbags/episode_3_2025-01-07-20-37-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2228014.log +started roslaunch server http://158.130.50.37:46133/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2228740] +process[print_text-11]: started with pid [2228741] +process[rosbag_record-12]: started with pid [2228742] +started bagging! +For topic gripper_width: timestamp difference is 103824726 for nearest: 1736300251469328276 - target: 1736300251365503550 at idx 63 +For topic gripper_width: timestamp difference is 107767357 for nearest: 1736300256869363560 - target: 1736300256761596203 at idx 140 +For topic gripper_width: timestamp difference is 108086533 for nearest: 1736300257669327683 - target: 1736300257561241150 at idx 151 +For topic gripper_width: timestamp difference is 109665816 for nearest: 1736300259669338172 - target: 1736300259559672356 at idx 171 +For topic gripper_width: timestamp difference is 103625098 for nearest: 1736300262669349232 - target: 1736300262565724134 at idx 211 +For topic gripper_width: timestamp difference is 116450801 for nearest: 1736300274069338336 - target: 1736300273952887535 at idx 249 +[INFO] [1736300275.646401]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736300276.415398]: Stored episode 4. +[INFO] [1736300276.427208]: Saving video... +[INFO] [1736300276.621277]: Video saved! +Starting episode 4 +[INFO] [1736300276.671903]: resetting environment +[INFO] [1736300276.698838]: cleared current plan +[INFO] [1736300276.704995]: moving to home +[INFO] [1736300280.316288]: reached home +[INFO] [1736300280.322257]: reset action history +[INFO] [1736300283.113316]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300290.294685]: resetting environment +[INFO] [1736300290.310128]: cleared current plan +[INFO] [1736300290.310396]: moving to home +[INFO] [1736300291.312276]: reached home +[INFO] [1736300291.316486]: reset action history +[INFO] [1736300294.092280]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/episode_rosbags/episode_4_2025-01-07-20-38-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2228014.log +started roslaunch server http://158.130.50.37:34277/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2228876] +process[print_text-14]: started with pid [2228877] +process[rosbag_record-15]: started with pid [2228878] +started bagging! +For topic gripper_width: timestamp difference is 117433405 for nearest: 1736300311569328880 - target: 1736300311451895475 at idx 228 +[INFO] [1736300322.871133]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736300323.662534]: Stored episode 5. +[INFO] [1736300323.664568]: Saving video... +[INFO] [1736300323.894860]: Video saved! +[INFO] [1736300323.961003]: resetting environment +[INFO] [1736300323.970549]: cleared current plan +[INFO] [1736300323.973510]: moving to home +[INFO] [1736300327.581203]: reached home +[INFO] [1736300327.588264]: reset action history +[INFO] [1736300330.428471]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/requirements.txt b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-metadata.json b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2091ba4a5b5318a043a9bea33a744b209d980bf3 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:34:23.442284", + "startedAt": "2025-01-08T01:34:23.013073", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3936.1075, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.041, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 593.6038970947266 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8afae20f77bc6369092aa973e77e8ce50b47c3e4 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "733d8003db33e86d4c9306deca409a10ad441137d5f571fd33956c260dcfaf23", "size": 1133946, "path": "media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4"}, "global_step": 665, "_timestamp": 1736300336.567505, "_runtime": 273.5465259552002, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "59fdd9b7330343a76e3c3136f94d1b903fb3f1bcbcd74a2013c23bb9714174be", "size": 1209518, "path": "media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4"}, "eval/num_success": 0.0, "episode": 55.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "8d54e64f7a05a499157babbc41d91945357bf22b215de5365e129c0888d9a842", "size": 1072150, "path": "media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f279f9c9369c3b87c444ccadd04c31defe965ead14f924a256b420181a8673da", "size": 1146457, "path": "media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "1a8eddcd55e59db846f73f4cacee5bd18cc056c973203e17c607c793ed2a372e", "size": 1159741, "path": "media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://zntwtbm9s4ijv2dsaiwuu53eh8qvooaux2tw4y2zk4uyildn422dfac50v7mdbx1kzyz2m89zr0315hrjpd9vujv74bvwrfu388stwstva17oru8hqicye7w34udf3bz/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://buejufc15zq9507lso1djgnerujkfwtgorykim9otzdoxvke2ym4r4l3eeuqtc0u6pcpv7oxtobe66zuf4kjawzopxomdlkwc8vgll7xyzwr8f2jl090pi0hzj6lmpbq:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 55, "total_success_rate": 0.45454545454545453, "total_success_rate_lb_0.01": 0.30244287852423735, "total_success_rate_ub_0.01": 0.6118613779321108, "total_success_rate_lb_0.025": 0.33041951808744774, "total_success_rate_ub_0.025": 0.5936293899825711, "total_success_rate_lb_0.05": 0.34962770345834293, "total_success_rate_ub_0.05": 0.5627395361536432, "total_success_rate_lb_0.1": 0.36513858291223766, "total_success_rate_ub_0.1": 0.5477405264878159, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "6f11d39d833a9161357e07f5fc45d0ab7e80f5d80fae57563f39b06f11417162", "size": 525, "artifact_path": "wandb-client-artifact://3yx1grq6cb7du7ernf0gnkixuoc73fez4cv9ay7rja5ht1jog9wvhjvtqmtnus98du98h5vbqhz1g6uql52p3un0355vokdlo4upeihj7lah13nd6bt0g44jut38huxn/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://s0921errpz6skmfaz37b3md708b7gcfwbxlt83nsgdj0492lqhuqhgc2qtvcr779pbyol50o1r3ahztn6bv2ox25j511pd1ocn7mnhi2bcjf9zfku7vnd0hv2ap2vuyh:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 286}, "eval/total_success_rate": 0.4545454680919647, "eval/total_success_rate_lb_0.01": 0.30244287848472595, "eval/total_success_rate_ub_0.01": 0.6118613481521606, "eval/total_success_rate_lb_0.025": 0.33041951060295105, "eval/total_success_rate_ub_0.025": 0.5936293601989746, "eval/total_success_rate_lb_0.05": 0.34962770342826843, "eval/total_success_rate_ub_0.05": 0.5627395510673523, "eval/total_success_rate_lb_0.1": 0.36513859033584595, "eval/total_success_rate_ub_0.1": 0.5477405190467834} \ No newline at end of file diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug-internal.log b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..902749567ea6335abd11b3e88328762ec17ac91b --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug-internal.log @@ -0,0 +1,798 @@ +2025-01-07 20:34:23,021 INFO StreamThr :2228203 [internal.py:wandb_internal():85] W&B internal server running at pid: 2228203, started at: 2025-01-07 20:34:23.020938 +2025-01-07 20:34:23,022 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:34:23,033 INFO WriterThread:2228203 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/run-2uvisx3r.wandb +2025-01-07 20:34:23,033 DEBUG SenderThread:2228203 [sender.py:send():379] send: header +2025-01-07 20:34:23,043 DEBUG SenderThread:2228203 [sender.py:send():379] send: run +2025-01-07 20:34:23,336 INFO SenderThread:2228203 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files +2025-01-07 20:34:23,336 INFO SenderThread:2228203 [sender.py:_start_run_threads():1188] run started: 2uvisx3r with start time 1736300063.020979 +2025-01-07 20:34:23,350 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:34:23,350 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:34:23,423 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:34:23,428 DEBUG HandlerThread:2228203 [system_info.py:__init__():26] System info init +2025-01-07 20:34:23,428 DEBUG HandlerThread:2228203 [system_info.py:__init__():41] System info init done +2025-01-07 20:34:23,428 INFO HandlerThread:2228203 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:34:23,428 INFO SystemMonitor:2228203 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:34:23,428 INFO HandlerThread:2228203 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:34:23,429 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:34:23,429 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:34:23,430 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:34:23,430 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:34:23,431 INFO SystemMonitor:2228203 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:34:23,442 DEBUG HandlerThread:2228203 [system_info.py:probe():152] Probing system +2025-01-07 20:34:23,443 DEBUG HandlerThread:2228203 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:34:23,448 DEBUG HandlerThread:2228203 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:34:23,448 DEBUG HandlerThread:2228203 [system_info.py:probe():200] Probing system done +2025-01-07 20:34:23,448 DEBUG HandlerThread:2228203 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:34:23.442284', 'startedAt': '2025-01-08T01:34:23.013073', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.1075, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.041, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 593.6038970947266}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:34:23,448 INFO HandlerThread:2228203 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:34:23,448 INFO HandlerThread:2228203 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:34:23,448 DEBUG HandlerThread:2228203 [system_info.py:_save_code():44] Saving code +2025-01-07 20:34:23,449 DEBUG HandlerThread:2228203 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:34:23,449 DEBUG HandlerThread:2228203 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:34:23,511 DEBUG HandlerThread:2228203 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:34:23,512 INFO HandlerThread:2228203 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:34:23,515 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:34:23,515 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:34:23,515 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:34:23,515 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:34:23,756 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:34:23,756 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:34:23,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:34:23,757 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:34:23,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:23,759 INFO HandlerThread:2228203 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/tb" + save: true +} +control { + relay_id: "a56d13a1fbde40a9bc9a1c91f550caa5" +} +_info { + stream_id: "2uvisx3r" +} + +2025-01-07 20:34:23,759 DEBUG HandlerThread:2228203 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:34:23,767 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/5u6zzksn-diff.patch +2025-01-07 20:34:23,778 INFO wandb-upload_0:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/02fvii0r-wandb-metadata.json +2025-01-07 20:34:23,817 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/rv78ku7a-code/FISH/eval_robot.py +2025-01-07 20:34:23,893 DEBUG SenderThread:2228203 [sender.py:send():379] send: telemetry +2025-01-07 20:34:23,893 DEBUG SenderThread:2228203 [sender.py:send():379] send: tbrecord +2025-01-07 20:34:23,894 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:34:23,894 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 with policy live +2025-01-07 20:34:24,084 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/yebrvrg8-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/requirements.txt +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code/FISH/eval_robot.py +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-metadata.json +2025-01-07 20:34:24,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/diff.patch +2025-01-07 20:34:24,339 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:24,339 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code +2025-01-07 20:34:24,339 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code/FISH +2025-01-07 20:34:24,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:25,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:26,338 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:26,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:27,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:28,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:28,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:29,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:30,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:31,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:32,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:33,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:33,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:34,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:35,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:36,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:37,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:38,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:34:38,757 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:34:38,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:38,894 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:39,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:40,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:41,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:42,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:43,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:44,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:44,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:45,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:46,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:47,179 DEBUG SenderThread:2228203 [sender.py:send():379] send: config +2025-01-07 20:34:47,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:48,344 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:48,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:49,758 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:49,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:50,344 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:50,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:51,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:52,345 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:52,760 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:53,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:34:53,758 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:34:53,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:54,345 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:54,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:55,662 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:34:55,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:56,346 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:56,346 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml +2025-01-07 20:34:56,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:57,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:58,347 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:34:58,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:34:59,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:00,348 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:00,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:01,679 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:01,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:02,348 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:02,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:03,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:04,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:05,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:06,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:06,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:07,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:08,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:35:08,757 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:35:08,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:09,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:10,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:11,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:11,835 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:12,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:13,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:14,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:15,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:16,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:17,759 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:17,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:18,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:19,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:20,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:21,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:22,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:22,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:23,431 DEBUG SystemMonitor:2228203 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:35:23,432 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:35:23,757 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:35:23,758 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:35:23,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:24,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:25,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:26,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:27,328 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:35:27,328 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 with policy now +2025-01-07 20:35:27,329 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:35:27,329 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:35:27,330 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:35:27,330 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos +2025-01-07 20:35:27,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:35:27,612 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/yrn5mbvj-media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 +2025-01-07 20:35:27,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:27,901 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/gnanrqee-media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 +2025-01-07 20:35:28,334 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:28,355 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:28,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:29,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:30,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:31,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:32,356 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:32,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:33,649 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:33,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:34,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:35,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:36,358 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:36,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:37,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:38,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:38,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:35:38,763 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:35:38,805 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:39,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:40,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:41,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:42,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:43,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:43,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:44,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:45,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:46,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:47,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:48,361 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:35:48,626 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/vav8km2y-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:35:48,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:49,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:49,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:50,362 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:50,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:51,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:52,363 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:52,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:53,432 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:35:53,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:35:53,761 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:35:53,809 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:54,364 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:35:54,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:54,908 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:35:55,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:56,786 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:57,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:58,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:35:59,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:00,762 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:00,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:01,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:02,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:03,779 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:04,783 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:05,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:05,782 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:06,782 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:07,782 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:08,371 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:08,761 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:36:08,761 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:36:08,805 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:09,783 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:10,783 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:10,920 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:11,798 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:12,798 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:13,798 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:14,801 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:15,802 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:16,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:16,809 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:17,810 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:18,810 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:19,810 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:20,814 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:20,945 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:36:20,954 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:36:20,955 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 with policy now +2025-01-07 20:36:20,955 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:36:20,955 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:36:20,955 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:36:21,222 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/pcq6w7qv-media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 +2025-01-07 20:36:21,375 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:36:21,375 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 +2025-01-07 20:36:21,376 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:36:21,647 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/8fj8v00n-media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 +2025-01-07 20:36:21,818 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:21,956 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:22,376 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:22,814 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:23,433 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:36:23,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:36:23,769 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:36:23,815 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:24,816 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:25,823 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:26,377 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:26,823 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:27,183 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:27,819 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:28,378 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:28,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:29,825 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:30,823 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:31,823 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:32,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:32,822 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:33,822 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:34,825 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:35,825 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:36,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:37,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:37,979 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:38,381 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:36:38,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:36:38,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:36:38,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:39,195 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:36:39,195 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:36:39,196 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:36:39,381 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:36:39,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:40,381 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:40,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:41,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:42,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:43,192 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:43,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:44,383 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:36:44,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:45,826 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:46,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:47,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:48,767 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:48,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:49,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:50,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:51,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:52,827 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:53,433 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:36:53,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:36:53,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:36:53,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:53,901 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:54,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:55,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:56,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:57,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:58,828 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:36:59,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:36:59,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:00,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:01,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:02,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:03,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:04,829 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:05,768 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:05,830 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:06,830 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:07,830 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:08,763 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:37:08,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:37:08,830 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:09,833 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:09,949 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:37:09,950 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:37:09,950 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 with policy now +2025-01-07 20:37:09,950 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:37:09,950 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:37:09,951 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:37:10,231 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/bf8wuoru-media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 +2025-01-07 20:37:10,391 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:37:10,391 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:10,391 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 +2025-01-07 20:37:10,392 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:37:10,652 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/sqnbzomx-media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 +2025-01-07 20:37:10,833 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:10,954 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:11,835 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:12,392 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:12,836 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:13,837 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:14,392 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:14,838 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:15,841 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:16,393 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:16,455 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:16,835 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:17,839 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:18,394 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:18,839 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:19,841 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:20,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:21,771 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:21,838 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:22,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:23,030 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:37:23,030 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:37:23,031 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:37:23,396 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:37:23,396 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:37:23,434 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:37:23,642 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/n0cvzznu-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:37:23,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:37:23,766 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:37:23,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:24,396 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:24,845 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:25,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:26,848 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:27,345 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:27,839 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:28,397 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:28,881 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:29,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:30,398 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:30,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:31,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:32,399 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:32,529 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:32,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:33,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:34,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:35,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:36,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:37,690 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:37,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:38,401 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:38,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:37:38,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:37:38,840 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:39,842 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:40,401 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:40,842 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:41,842 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:42,402 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:42,721 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:42,842 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:43,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:44,403 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:44,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:45,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:46,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:47,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:47,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:48,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:49,843 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:50,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:51,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:52,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:52,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:53,434 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:37:53,764 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:37:53,764 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:37:53,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:54,844 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:55,847 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:56,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:56,657 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:37:56,657 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 with policy now +2025-01-07 20:37:56,664 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:37:56,665 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:37:56,665 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:37:56,665 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:37:56,847 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:56,913 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/1oh8qdpv-media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 +2025-01-07 20:37:57,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:37:57,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 +2025-01-07 20:37:57,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:37:57,687 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/smlnlylj-media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 +2025-01-07 20:37:57,852 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:58,407 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:37:58,707 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:37:58,851 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:37:59,849 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:00,861 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:01,861 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:02,409 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:02,872 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:03,777 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:03,856 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:04,410 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:04,854 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:05,861 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:06,861 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:07,862 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:08,770 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:38:08,770 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:38:08,868 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:08,888 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:09,867 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:10,412 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:38:10,862 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:11,089 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:11,089 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:11,089 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:11,412 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:11,862 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:12,412 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:12,866 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:13,876 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:14,092 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:14,862 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:15,863 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:16,414 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:16,863 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:17,864 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:18,863 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:19,771 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:19,867 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:20,884 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:21,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:22,877 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:23,435 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:38:23,769 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:38:23,769 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:38:23,877 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:24,877 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:24,921 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:25,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:26,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:27,879 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:28,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:29,879 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:30,772 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:30,878 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:31,880 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:32,419 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:32,881 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:33,880 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:34,880 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:35,772 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:35,880 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:36,881 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:37,887 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:38,766 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:38:38,766 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:38:38,885 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:39,886 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:40,875 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:40,887 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:41,886 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:42,913 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:43,895 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:43,956 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:38:43,956 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 with policy now +2025-01-07 20:38:43,960 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:38:43,961 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:43,961 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:43,962 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:44,218 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/ri231522-media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 +2025-01-07 20:38:44,423 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:44,423 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:44,423 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 +2025-01-07 20:38:44,424 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval +2025-01-07 20:38:44,730 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/73bz6bw5-media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 +2025-01-07 20:38:44,899 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:45,903 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:45,978 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:46,424 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:46,906 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:47,906 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:48,424 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:48,907 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:49,914 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:50,911 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:51,473 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:51,911 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:52,426 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:52,906 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:53,435 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:38:53,773 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:38:53,773 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:38:53,917 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:54,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:55,138 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:55,139 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:55,140 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:55,427 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:55,427 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:38:55,674 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/rpdg1y7x-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:38:55,913 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:56,027 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:38:56,027 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:38:56,193 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:38:56,427 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:56,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:38:56,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media +2025-01-07 20:38:56,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval +2025-01-07 20:38:56,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table +2025-01-07 20:38:56,510 INFO wandb-upload_2:2228203 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp5svb0nm +2025-01-07 20:38:56,633 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/h9t81edn-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:38:56,684 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:38:56,693 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:38:56,901 INFO SenderThread:2228203 [sender.py:send_request_log_artifact():1525] logged artifact run-2uvisx3r-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQwNDIyMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzNjEyOA==', 'latestArtifact': None}} +2025-01-07 20:38:56,901 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:38:56,901 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:38:56,901 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:56,901 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,901 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:38:56,902 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,902 DEBUG SenderThread:2228203 [sender.py:send():379] send: telemetry +2025-01-07 20:38:56,902 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,903 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,903 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,903 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,905 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,905 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,905 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,906 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,906 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,906 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,906 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,907 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,907 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,907 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,907 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,907 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,908 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,908 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,908 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,908 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,909 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:56,909 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:56,909 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:38:56,912 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:57,152 INFO wandb-upload_0:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/3v8spda0-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:38:57,320 INFO wandb-upload_1:2228203 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprply37rq +2025-01-07 20:38:57,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:57,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:38:57,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json +2025-01-07 20:38:57,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval +2025-01-07 20:38:57,706 INFO SenderThread:2228203 [sender.py:send_request_log_artifact():1525] logged artifact run-2uvisx3r-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQwNDI0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzNjEzMQ==', 'latestArtifact': None}} +2025-01-07 20:38:57,707 DEBUG SenderThread:2228203 [sender.py:send():379] send: files +2025-01-07 20:38:57,707 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json with policy now +2025-01-07 20:38:57,707 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:38:57,707 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:38:57,708 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:38:57,883 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/pa6cqtzh-media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json +2025-01-07 20:38:57,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:58,131 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/vdn7kbpn-media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json +2025-01-07 20:38:58,428 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:38:58,429 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:38:58,913 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:38:59,913 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:00,914 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:01,914 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:02,773 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:02,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:03,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:04,914 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:05,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:06,915 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:07,773 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:07,917 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:08,765 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:39:08,766 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:39:08,916 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:09,553 DEBUG SenderThread:2228203 [sender.py:send():379] send: exit +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:send_exit():588] handling runtime: 286 +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:send_exit():594] send defer +2025-01-07 20:39:09,554 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,554 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:39:09,554 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:09,554 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:39:09,555 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:39:09,555 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,555 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:39:09,555 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:09,555 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:39:09,555 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:39:09,555 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,555 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:39:09,555 INFO HandlerThread:2228203 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:39:09,555 DEBUG SystemMonitor:2228203 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:39:09,555 DEBUG SystemMonitor:2228203 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:39:09,556 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:39:09,556 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:39:09,611 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:39:09,612 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:39:09,612 INFO HandlerThread:2228203 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:39:09,612 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:09,612 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:39:09,612 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:39:09,612 DEBUG SenderThread:2228203 [sender.py:send():379] send: stats +2025-01-07 20:39:09,613 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,613 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:39:09,613 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:09,613 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:39:09,613 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:39:09,613 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:09,613 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:39:10,433 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:39:10,433 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:39:14,435 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml +2025-01-07 20:39:16,149 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:16,149 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:16,149 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:39:16,149 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,150 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:39:16,150 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:39:16,150 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:16,150 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,151 DEBUG SenderThread:2228203 [sender.py:send():379] send: history +2025-01-07 20:39:16,151 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:39:16,151 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:39:16,152 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:39:16,152 DEBUG SenderThread:2228203 [sender.py:send():379] send: summary +2025-01-07 20:39:16,155 INFO SenderThread:2228203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:39:16,156 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,156 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:39:16,156 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:39:16,156 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,156 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:39:16,156 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,156 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:39:16,156 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:39:16,156 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:16,156 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,156 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:39:16,157 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,157 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:39:16,235 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:39:16,235 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,235 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:39:16,235 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,235 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:39:16,235 INFO SenderThread:2228203 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:39:16,238 INFO SenderThread:2228203 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:39:16,240 INFO SenderThread:2228203 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:39:16,249 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:39:16,249 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:16,249 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:39:16,250 DEBUG SenderThread:2228203 [sender.py:send():379] send: artifact +2025-01-07 20:39:16,436 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:39:16,436 INFO Thread-12 :2228203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:39:16,449 INFO SenderThread:2228203 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:39:16,449 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:16,450 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:39:16,450 INFO SenderThread:2228203 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:39:16,555 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:17,436 INFO SenderThread:2228203 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml config.yaml +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json wandb-summary.json +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/requirements.txt requirements.txt +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log output.log +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/diff.patch diff.patch +2025-01-07 20:39:17,437 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:39:17,439 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 media/videos/eval/1_eval_1_59fdd9b7330343a76e3c.mp4 +2025-01-07 20:39:17,439 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 media/videos/eval/4_eval_4_1a8eddcd55e59db846f7.mp4 +2025-01-07 20:39:17,440 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 media/videos/eval/2_eval_2_8d54e64f7a05a499157b.mp4 +2025-01-07 20:39:17,440 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 media/videos/eval/3_eval_3_f279f9c9369c3b87c444.mp4 +2025-01-07 20:39:17,440 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 media/videos/eval/0_eval_0_733d8003db33e86d4c93.mp4 +2025-01-07 20:39:17,444 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json media/table/eval/total_success_rate_ci_6_6f11d39d833a9161357e.table.json +2025-01-07 20:39:17,444 INFO SenderThread:2228203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:39:17,444 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:39:17,444 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:17,444 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:17,444 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:39:17,446 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:17,446 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:39:17,446 INFO SenderThread:2228203 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:39:17,555 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:17,555 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:17,636 INFO wandb-upload_2:2228203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/config.yaml +2025-01-07 20:39:17,655 INFO wandb-upload_0:2228203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/requirements.txt +2025-01-07 20:39:17,670 INFO wandb-upload_3:2228203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/wandb-summary.json +2025-01-07 20:39:17,672 INFO wandb-upload_4:2228203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/files/output.log +2025-01-07 20:39:17,691 INFO wandb-upload_1:2228203 [upload_job.py:push():130] Uploaded file /tmp/tmptoilyydbwandb/q2lcdjtn-events.out.tfevents.1736300063.leonmkim-ROG-Strix-G15CS-G15CS.2228014.0 +2025-01-07 20:39:17,892 INFO Thread-11 (_thread_body):2228203 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:39:17,892 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:17,892 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:39:17,892 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:17,892 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:39:17,892 INFO SenderThread:2228203 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:39:17,893 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:39:17,894 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:17,894 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:39:17,894 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:17,894 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:39:17,894 INFO SenderThread:2228203 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:39:18,115 INFO SenderThread:2228203 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:39:18,115 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:39:18,116 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:18,116 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:39:18,116 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:18,116 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:39:18,116 INFO SenderThread:2228203 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:39:18,116 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:39:18,116 INFO HandlerThread:2228203 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:39:18,116 DEBUG SenderThread:2228203 [sender.py:send():379] send: final +2025-01-07 20:39:18,116 DEBUG SenderThread:2228203 [sender.py:send():379] send: footer +2025-01-07 20:39:18,116 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: defer +2025-01-07 20:39:18,116 INFO SenderThread:2228203 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:39:18,117 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:18,117 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:39:18,117 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:39:18,117 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:39:18,118 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:39:18,118 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:18,118 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:39:18,118 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:18,172 DEBUG SenderThread:2228203 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:39:18,172 INFO MainThread:2228203 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:39:18,172 INFO MainThread:2228203 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:39:18,173 INFO MainThread:2228203 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:39:18,173 DEBUG HandlerThread:2228203 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:39:18,173 INFO HandlerThread:2228203 [handler.py:finish():882] shutting down handler +2025-01-07 20:39:19,118 INFO WriterThread:2228203 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/run-2uvisx3r.wandb +2025-01-07 20:39:19,172 INFO SenderThread:2228203 [sender.py:finish():1615] shutting down sender +2025-01-07 20:39:19,172 INFO SenderThread:2228203 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:39:19,172 INFO SenderThread:2228203 [file_pusher.py:join():175] waiting for file pusher diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug.log b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..13271972d84400633b193c758bf38e23f05e0143 --- /dev/null +++ b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Configure stats pid to 2228014 +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/settings +2025-01-07 20:34:23,014 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug.log +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/logs/debug-internal.log +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r'} +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:init():619] starting backend +2025-01-07 20:34:23,015 INFO MainThread:2228014 [wandb_init.py:init():623] setting up manager +2025-01-07 20:34:23,020 INFO MainThread:2228014 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:34:23,020 INFO MainThread:2228014 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:34:23,032 INFO MainThread:2228014 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:34:23,040 INFO MainThread:2228014 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:34:23,349 INFO MainThread:2228014 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:34:23,418 INFO MainThread:2228014 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:34:23,418 INFO MainThread:2228014 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:34:23,757 INFO MainThread:2228014 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:34:23,757 INFO MainThread:2228014 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:34:23,757 INFO MainThread:2228014 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:34:23,757 INFO MainThread:2228014 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:34:23,758 INFO MainThread:2228014 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:34:23,759 INFO MainThread:2228014 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2uvisx3r/tb, True +2025-01-07 20:34:47,179 INFO MainThread:2228014 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:39:19,174 WARNING MsgRouterThr:2228014 [router.py:message_loop():77] message_loop has been closed diff --git a/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/run-2uvisx3r.wandb b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/run-2uvisx3r.wandb new file mode 100644 index 0000000000000000000000000000000000000000..183892bbfe4dc679c70046e0383d60c085571a18 Binary files /dev/null and b/2uvisx3r/wandb/run-20250107_203423-2uvisx3r/run-2uvisx3r.wandb differ diff --git a/4tlpd4kc/.hydra/config.yaml b/4tlpd4kc/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/4tlpd4kc/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/4tlpd4kc/.hydra/hydra.yaml b/4tlpd4kc/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f35debeee16e0b0222f3f74cda3371dfd6f624e --- /dev/null +++ b/4tlpd4kc/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/4tlpd4kc + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/4tlpd4kc/.hydra/overrides.yaml b/4tlpd4kc/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/4tlpd4kc/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/4tlpd4kc/eval_policy.log b/4tlpd4kc/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..27cfc9afb6ef9be8304ab532154c9eda76fe8af9 --- /dev/null +++ b/4tlpd4kc/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 17:55:14,456][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:55:14,459][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 17:55:15,601][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2002149a914c06f256ecda4cb79b9c7eef5b96d --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 1 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1022_0' +wandb_notes: 1022_0_req_1348_0restarted_2 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 5 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/dq0rnfzj/.hydra/config.yaml b/dq0rnfzj/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/dq0rnfzj/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/dq0rnfzj/.hydra/hydra.yaml b/dq0rnfzj/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d824fc77942068b0bf3e8c132afc8623d1246d8a --- /dev/null +++ b/dq0rnfzj/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/dq0rnfzj + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/dq0rnfzj/.hydra/overrides.yaml b/dq0rnfzj/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/dq0rnfzj/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/dq0rnfzj/eval_policy.log b/dq0rnfzj/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..8a789837d7fc11b4bdb001f8fb8b2b4a11074649 --- /dev/null +++ b/dq0rnfzj/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 17:28:34,786][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:28:34,789][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 17:28:35,870][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/e3yh61zm/.hydra/config.yaml b/e3yh61zm/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/e3yh61zm/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/e3yh61zm/.hydra/hydra.yaml b/e3yh61zm/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0582d95fec796d151de505de81dbc867ddd14e57 --- /dev/null +++ b/e3yh61zm/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/e3yh61zm/.hydra/overrides.yaml b/e3yh61zm/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/e3yh61zm/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/e3yh61zm/episode_rosbags/aligned_depth_to_color_K.npy b/e3yh61zm/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/e3yh61zm/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/e3yh61zm/episode_rosbags/cam_tf_world.npy b/e3yh61zm/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/e3yh61zm/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/e3yh61zm/episode_rosbags/color_K.npy b/e3yh61zm/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/e3yh61zm/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/e3yh61zm/episode_rosbags/depth_K.npy b/e3yh61zm/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/e3yh61zm/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/e3yh61zm/eval_robot.log b/e3yh61zm/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..b68c9c58751830edbac48515c5da76f324f7c61f --- /dev/null +++ b/e3yh61zm/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 17:55:23,701][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:55:23,704][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/e3yh61zm/eval_video/0_eval.mp4 b/e3yh61zm/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f5b059a86447f32077cbd3745930a89cda1b8cd4 --- /dev/null +++ b/e3yh61zm/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41d66fcf0a834c3bfb0b3b676bc76587709b7a60cac9e86c4f9dfc88334481c9 +size 1130489 diff --git a/e3yh61zm/eval_video/1_eval.mp4 b/e3yh61zm/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a346a85d8234ae9f182e15894f774b0405f3d560 --- /dev/null +++ b/e3yh61zm/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27d2fca28dbf07e630def064d52cff1e4dec7c4ef45f7630a00451861ed77dc9 +size 1131474 diff --git a/e3yh61zm/eval_video/2_eval.mp4 b/e3yh61zm/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f52b74ab16916d27ab82fbc6b0b4b8357e35cff7 --- /dev/null +++ b/e3yh61zm/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abcd97352dad09227ec188da612ffee0d94faf3dc73cc0fcc66c51924c302ee +size 1145346 diff --git a/e3yh61zm/eval_video/3_eval.mp4 b/e3yh61zm/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..165c8fd866f77ebf34746d39585a8dd5e6183932 --- /dev/null +++ b/e3yh61zm/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12714e5cc65004ce4bac21af77d3fede418b9d7dd29150b35b5b30756951ad38 +size 1153288 diff --git a/e3yh61zm/eval_video/4_eval.mp4 b/e3yh61zm/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e3835ee478e12436b3e65a9c9d359d457946124 --- /dev/null +++ b/e3yh61zm/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c9f5cbe3bf7ff9da1fdc97ea75d4e415b25f10928bed9131407f7f6624f53eb +size 1155936 diff --git a/e3yh61zm/tb/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 b/e3yh61zm/tb/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 new file mode 100644 index 0000000000000000000000000000000000000000..02ea09c44032f10db5cf4286a1dfc81a9118eb6b --- /dev/null +++ b/e3yh61zm/tb/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:479930cf967cc3aa7d7ae3e878f790257e3a1b38124a2b0832025aa7a47052d3 +size 6539 diff --git a/e3yh61zm/wandb/debug-internal.log b/e3yh61zm/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..03d7cbbbdb5048822b5fc08421b72ca8c4f999fe --- /dev/null +++ b/e3yh61zm/wandb/debug-internal.log @@ -0,0 +1,784 @@ +2025-01-07 17:55:29,467 INFO StreamThr :2205467 [internal.py:wandb_internal():85] W&B internal server running at pid: 2205467, started at: 2025-01-07 17:55:29.466724 +2025-01-07 17:55:29,468 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:55:29,479 INFO WriterThread:2205467 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/run-e3yh61zm.wandb +2025-01-07 17:55:29,480 DEBUG SenderThread:2205467 [sender.py:send():379] send: header +2025-01-07 17:55:29,487 DEBUG SenderThread:2205467 [sender.py:send():379] send: run +2025-01-07 17:55:29,751 INFO SenderThread:2205467 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files +2025-01-07 17:55:29,752 INFO SenderThread:2205467 [sender.py:_start_run_threads():1188] run started: e3yh61zm with start time 1736290529.467055 +2025-01-07 17:55:29,770 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:55:29,770 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:55:29,815 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:55:29,820 DEBUG HandlerThread:2205467 [system_info.py:__init__():26] System info init +2025-01-07 17:55:29,820 DEBUG HandlerThread:2205467 [system_info.py:__init__():41] System info init done +2025-01-07 17:55:29,820 INFO HandlerThread:2205467 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:55:29,820 INFO SystemMonitor:2205467 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:55:29,820 INFO HandlerThread:2205467 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:55:29,820 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:55:29,822 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:55:29,822 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:55:29,822 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:55:29,823 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:55:29,832 DEBUG HandlerThread:2205467 [system_info.py:probe():152] Probing system +2025-01-07 17:55:29,836 DEBUG HandlerThread:2205467 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:55:29,842 DEBUG HandlerThread:2205467 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:55:29,842 DEBUG HandlerThread:2205467 [system_info.py:probe():200] Probing system done +2025-01-07 17:55:29,842 DEBUG HandlerThread:2205467 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:55:29.833044', 'startedAt': '2025-01-07T22:55:29.452951', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.324375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.25, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 789.7138137817383}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:55:29,843 INFO HandlerThread:2205467 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:55:29,843 INFO HandlerThread:2205467 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:55:29,843 DEBUG HandlerThread:2205467 [system_info.py:_save_code():44] Saving code +2025-01-07 17:55:29,848 DEBUG HandlerThread:2205467 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:55:29,848 DEBUG HandlerThread:2205467 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:55:29,910 DEBUG HandlerThread:2205467 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:55:29,910 INFO HandlerThread:2205467 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:55:29,914 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:55:29,914 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:55:29,914 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:55:29,915 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:55:30,158 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:55:30,158 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:55:30,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:55:30,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:55:30,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:30,162 INFO HandlerThread:2205467 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/tb" + save: true +} +control { + relay_id: "f58c3c22ca26407d829374d72f14e3ed" +} +_info { + stream_id: "e3yh61zm" +} + +2025-01-07 17:55:30,162 DEBUG HandlerThread:2205467 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:55:30,170 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/4zl12u4h-wandb-metadata.json +2025-01-07 17:55:30,188 INFO wandb-upload_1:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/khtzvqiv-code/FISH/eval_robot.py +2025-01-07 17:55:30,236 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/ovwg34yv-diff.patch +2025-01-07 17:55:30,247 DEBUG SenderThread:2205467 [sender.py:send():379] send: telemetry +2025-01-07 17:55:30,247 DEBUG SenderThread:2205467 [sender.py:send():379] send: tbrecord +2025-01-07 17:55:30,247 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:55:30,247 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 with policy live +2025-01-07 17:55:30,441 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/5faioorm-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:55:30,753 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/requirements.txt +2025-01-07 17:55:30,753 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code/FISH/eval_robot.py +2025-01-07 17:55:30,753 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-metadata.json +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/diff.patch +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code/FISH +2025-01-07 17:55:31,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:32,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:32,752 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:33,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:33,508 DEBUG SenderThread:2205467 [sender.py:send():379] send: config +2025-01-07 17:55:34,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:34,515 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:34,753 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:35,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:36,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:36,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:37,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:38,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:38,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:39,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:40,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:40,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:40,755 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:41,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:42,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:42,756 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:43,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:44,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:45,158 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:55:45,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:55:45,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:45,244 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:46,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:47,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:48,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:49,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:50,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:50,758 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:51,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:51,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:52,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:53,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:54,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:54,759 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:55,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:56,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:56,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:57,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:58,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:59,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:00,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:56:00,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:56:00,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:01,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:01,254 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:01,761 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml +2025-01-07 17:56:02,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:03,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:04,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:05,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:06,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:07,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:07,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:08,164 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:09,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:10,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:11,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:12,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:12,764 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:13,029 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:13,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:14,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:15,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:56:15,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:56:15,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:16,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:17,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:18,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:18,255 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:19,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:20,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:20,349 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:56:20,350 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:56:20,350 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 with policy now +2025-01-07 17:56:20,350 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:56:20,350 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:56:20,351 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:56:20,638 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/gm9moaxi-media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:56:20,767 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media +2025-01-07 17:56:21,045 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/wz1rsoth-media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 +2025-01-07 17:56:21,167 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:22,167 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:22,767 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:23,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:23,351 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:24,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:25,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:26,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:26,768 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:27,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:28,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:28,769 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:28,785 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:29,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:29,823 DEBUG SystemMonitor:2205467 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:56:29,824 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:56:30,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:56:30,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:56:30,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:31,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:32,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:32,770 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:56:33,063 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/bsfb9bdp-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:56:33,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:34,041 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:34,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:34,771 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:35,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:36,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:36,771 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:37,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:38,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:38,772 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:39,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:39,941 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:40,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:41,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:42,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:42,774 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:43,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:44,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:44,775 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:45,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:56:45,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:56:45,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:45,320 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:46,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:46,776 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:47,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:48,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:49,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:50,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:50,777 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:51,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:51,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:52,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:52,778 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:53,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:54,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:55,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:56,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:56,173 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:56,779 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:57,173 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:58,173 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:58,780 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:59,175 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:59,824 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:57:00,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:57:00,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:57:00,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:01,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:01,263 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:02,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:02,781 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:03,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:04,014 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:57:04,014 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 with policy now +2025-01-07 17:57:04,017 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:57:04,017 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:57:04,017 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:57:04,018 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:57:04,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:04,278 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/82pss7d7-media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 +2025-01-07 17:57:04,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:57:04,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:04,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 +2025-01-07 17:57:04,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:57:05,085 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/ojwi1mgs-media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 +2025-01-07 17:57:05,180 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:06,175 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:06,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:07,167 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:07,180 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:08,178 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:08,783 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:09,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:10,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:11,184 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:12,184 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:12,569 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:12,784 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:13,184 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:14,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:15,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:57:15,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:57:15,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:16,184 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:17,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:18,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:18,271 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:19,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:20,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:21,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:22,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:23,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:24,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:24,186 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:25,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:25,573 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:57:25,573 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:57:25,574 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:57:25,788 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:57:25,789 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:57:26,186 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:26,789 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:27,186 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:28,186 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:28,790 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:29,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:29,825 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:57:29,825 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:30,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:57:30,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:57:30,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:30,790 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:31,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:32,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:33,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:34,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:35,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:35,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:36,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:37,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:38,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:39,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:40,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:41,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:41,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:42,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:43,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:44,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:45,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:57:45,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:57:45,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:46,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:46,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:47,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:48,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:49,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:50,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:51,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:52,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:52,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:53,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:54,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:55,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:56,193 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:56,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:56,809 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:57:56,809 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 with policy now +2025-01-07 17:57:56,812 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:57:56,812 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:57:56,812 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:57:56,812 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:57:57,098 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/ft0q8hsn-media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 +2025-01-07 17:57:57,210 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:57,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:57:57,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 +2025-01-07 17:57:57,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:57:57,838 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:58,144 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/z9w04c9d-media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 +2025-01-07 17:57:58,206 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:58,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:59,204 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:59,825 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:58:00,176 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:58:00,176 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:58:00,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:01,208 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:02,208 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:02,801 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:03,216 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:03,331 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:04,213 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:04,801 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:05,213 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:06,215 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:06,403 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:58:06,403 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:58:06,404 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:58:06,802 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:58:06,802 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:58:06,802 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:07,074 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/peeu4rwv-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:58:07,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:08,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:08,803 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:08,929 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:09,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:10,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:10,803 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:11,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:12,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:12,804 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:13,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:14,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:14,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:15,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:58:15,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:58:15,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:16,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:17,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:18,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:19,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:19,262 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:20,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:21,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:22,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:23,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:24,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:25,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:25,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:26,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:27,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:28,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:29,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:29,826 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:58:30,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:58:30,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:58:30,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:30,259 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:30,811 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:31,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:32,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:33,220 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:34,220 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:35,220 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:36,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:36,220 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:37,222 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:38,118 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:58:38,118 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 with policy now +2025-01-07 17:58:38,118 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:58:38,119 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:58:38,119 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:58:38,121 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:58:38,221 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:38,387 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/2a2zlhgh-media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 +2025-01-07 17:58:38,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:58:38,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:38,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 +2025-01-07 17:58:38,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:58:39,099 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/49oiyhb6-media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 +2025-01-07 17:58:39,230 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:40,236 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:40,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:41,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:41,234 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:42,230 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:42,815 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:43,233 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:44,229 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:45,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:58:45,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:58:45,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:46,234 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:46,264 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:46,816 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:47,242 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:48,236 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:48,455 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:58:48,456 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:58:48,457 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:58:48,817 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:58:48,817 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:58:48,818 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:49,236 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:50,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:50,818 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:51,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:51,752 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:52,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:52,819 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:53,237 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:54,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:54,820 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:55,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:56,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:57,176 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:57,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:58,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:59,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:59,826 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:59:00,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:59:00,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:59:00,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:01,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:02,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:02,268 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:03,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:04,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:05,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:06,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:07,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:07,381 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:08,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:08,825 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:09,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:10,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:11,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:12,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:13,194 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:13,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:14,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:14,827 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:15,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:59:15,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:59:15,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:16,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:17,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:18,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:18,241 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:19,248 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:19,965 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:59:19,965 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 with policy now +2025-01-07 17:59:19,966 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:59:19,966 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:59:19,966 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:19,968 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:20,230 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/bdve6x6w-media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 +2025-01-07 17:59:20,243 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:20,829 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:59:20,830 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:20,830 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 +2025-01-07 17:59:20,830 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:59:21,150 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/pib94gwe-media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 +2025-01-07 17:59:21,244 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:22,244 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:23,251 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:23,696 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:24,250 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:24,831 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:25,250 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:26,248 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:27,318 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:28,296 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:28,832 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:29,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:29,597 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:29,826 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:59:30,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:59:30,169 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:59:30,306 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:31,305 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:32,304 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:32,513 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:59:32,514 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:32,514 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:32,811 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:59:32,811 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:59:32,834 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:59:32,834 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:59:32,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:59:32,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media +2025-01-07 17:59:32,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval +2025-01-07 17:59:32,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table +2025-01-07 17:59:32,949 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:59:33,049 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/x3wsjfcs-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:59:33,094 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/hnlwicwb-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:59:33,296 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:33,326 INFO wandb-upload_1:2205467 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4lzqx5y_ +2025-01-07 17:59:33,370 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:59:33,413 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:59:33,695 INFO SenderThread:2205467 [sender.py:send_request_log_artifact():1525] logged artifact run-e3yh61zm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAzMTk4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTczMTgzNA==', 'latestArtifact': None}} +2025-01-07 17:59:33,695 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:59:33,695 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 17:59:33,696 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:59:33,696 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,696 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,696 DEBUG SenderThread:2205467 [sender.py:send():379] send: telemetry +2025-01-07 17:59:33,697 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,697 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,697 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,698 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,698 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,700 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,700 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,700 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,701 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,701 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,701 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,701 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,701 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,702 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,702 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,702 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,702 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,703 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,703 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,703 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,703 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,704 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,704 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:59:33,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:59:33,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:59:33,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:59:33,836 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval +2025-01-07 17:59:33,955 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/tgj5o8az-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:59:34,029 INFO wandb-upload_1:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/uyc9lbf6-media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:59:34,076 INFO wandb-upload_0:2205467 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0ldto3x9 +2025-01-07 17:59:34,296 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:34,535 INFO SenderThread:2205467 [sender.py:send_request_log_artifact():1525] logged artifact run-e3yh61zm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAzMjAxMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTczMTgzNg==', 'latestArtifact': None}} +2025-01-07 17:59:34,535 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:59:34,535 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json with policy now +2025-01-07 17:59:34,536 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:59:34,536 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:34,536 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:34,810 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/fmwovnj0-media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:59:34,836 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:59:34,836 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:35,176 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:35,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:36,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:37,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:38,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:39,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:40,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:41,176 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:41,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:42,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:43,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:44,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:45,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:59:45,161 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:59:45,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:46,288 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:46,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:47,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:48,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:49,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:50,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:51,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:52,180 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:52,304 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:52,843 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml +2025-01-07 17:59:53,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:54,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:55,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:56,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:57,300 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:58,177 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:58,300 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:59,300 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:59,827 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 18:00:00,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:00:00,161 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:00:00,300 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:00,873 DEBUG SenderThread:2205467 [sender.py:send():379] send: exit +2025-01-07 18:00:00,873 INFO SenderThread:2205467 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:send_exit():588] handling runtime: 271 +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:send_exit():594] send defer +2025-01-07 18:00:00,874 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,874 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:00:00,874 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:00:00,875 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:00:00,875 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:00,875 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:00:00,875 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:00:00,875 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:00:00,876 DEBUG SystemMonitor:2205467 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:00:00,876 DEBUG SystemMonitor:2205467 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:00:00,926 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:00:00,926 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:00:00,926 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:00:00,927 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:00,927 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:00:00,927 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:00:00,927 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 18:00:00,927 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,927 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:00:00,927 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:00,927 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:00:00,927 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:00:00,928 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,928 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:00:01,846 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 18:00:02,847 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 18:00:07,522 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:07,522 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:07,522 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:00:07,523 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:07,523 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:00:07,524 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:00:07,524 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:07,524 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 18:00:07,524 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:00:07,524 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:07,524 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:00:07,525 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:00:07,525 DEBUG SenderThread:2205467 [sender.py:send():379] send: summary +2025-01-07 18:00:07,529 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:00:07,529 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:07,529 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:00:07,529 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:00:07,529 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:07,529 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:00:07,530 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:07,530 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:00:07,530 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:00:07,530 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:07,530 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:07,530 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:00:07,530 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:07,530 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:00:07,849 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 18:00:07,875 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:10,151 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:00:10,151 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:10,151 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:10,152 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:00:10,152 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:10,152 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:00:10,152 INFO SenderThread:2205467 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:00:10,154 INFO SenderThread:2205467 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:00:10,156 INFO SenderThread:2205467 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:00:10,170 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:00:10,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:10,170 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:00:10,170 DEBUG SenderThread:2205467 [sender.py:send():379] send: artifact +2025-01-07 18:00:10,384 INFO SenderThread:2205467 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:00:10,384 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:10,384 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:00:10,384 INFO SenderThread:2205467 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml config.yaml +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json wandb-summary.json +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/requirements.txt requirements.txt +2025-01-07 18:00:10,851 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:00:10,851 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 18:00:10,851 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log output.log +2025-01-07 18:00:10,851 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/diff.patch diff.patch +2025-01-07 18:00:10,852 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 +2025-01-07 18:00:10,859 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 18:00:10,859 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:00:10,859 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:00:10,859 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:10,859 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:00:10,860 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:10,860 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:00:10,860 INFO SenderThread:2205467 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:00:10,876 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:10,876 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:11,035 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 18:00:11,046 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml +2025-01-07 18:00:11,077 INFO wandb-upload_1:2205467 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/requirements.txt +2025-01-07 18:00:11,093 INFO wandb-upload_4:2205467 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 18:00:11,181 INFO wandb-upload_3:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/f4obak7k-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 18:00:11,381 INFO Thread-11 (_thread_body):2205467 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:00:11,381 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:11,381 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:00:11,381 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:11,382 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:00:11,382 INFO SenderThread:2205467 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:00:11,383 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:00:11,383 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:11,383 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:00:11,383 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:11,383 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:00:11,383 INFO SenderThread:2205467 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:00:11,649 INFO SenderThread:2205467 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:00:11,649 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:00:11,649 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:11,649 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:00:11,649 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:11,649 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:00:11,650 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:00:11,650 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:11,650 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:00:11,650 DEBUG SenderThread:2205467 [sender.py:send():379] send: final +2025-01-07 18:00:11,650 DEBUG SenderThread:2205467 [sender.py:send():379] send: footer +2025-01-07 18:00:11,650 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:11,650 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:00:11,651 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:11,651 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:11,651 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:00:11,651 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:00:11,651 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:11,653 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:00:11,653 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:00:11,654 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:11,717 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:11,717 INFO MainThread:2205467 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:00:11,717 INFO MainThread:2205467 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:00:11,718 INFO MainThread:2205467 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:00:11,718 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:00:11,718 INFO HandlerThread:2205467 [handler.py:finish():882] shutting down handler +2025-01-07 18:00:12,653 INFO WriterThread:2205467 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/run-e3yh61zm.wandb +2025-01-07 18:00:12,717 INFO SenderThread:2205467 [sender.py:finish():1615] shutting down sender +2025-01-07 18:00:12,717 INFO SenderThread:2205467 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:00:12,717 INFO SenderThread:2205467 [file_pusher.py:join():175] waiting for file pusher diff --git a/e3yh61zm/wandb/debug.log b/e3yh61zm/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..61b24a4763edba4efec4f633bd283f768429181c --- /dev/null +++ b/e3yh61zm/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Configure stats pid to 2205299 +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/settings +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug.log +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug-internal.log +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm'} +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:init():619] starting backend +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:init():623] setting up manager +2025-01-07 17:55:29,465 INFO MainThread:2205299 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:55:29,466 INFO MainThread:2205299 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:55:29,479 INFO MainThread:2205299 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:55:29,485 INFO MainThread:2205299 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:55:29,770 INFO MainThread:2205299 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:55:29,810 INFO MainThread:2205299 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:55:29,811 INFO MainThread:2205299 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:55:30,158 INFO MainThread:2205299 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:55:30,159 INFO MainThread:2205299 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:55:30,159 INFO MainThread:2205299 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:55:30,159 INFO MainThread:2205299 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:55:30,161 INFO MainThread:2205299 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:55:30,162 INFO MainThread:2205299 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/tb, True +2025-01-07 17:55:33,507 INFO MainThread:2205299 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:00:12,719 WARNING MsgRouterThr:2205299 [router.py:message_loop():77] message_loop has been closed diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code/FISH/eval_robot.py b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33f3d3751fb868390325d13c07ab763e8ea6823a --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736290529 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/diff.patch b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a994978ad1dd8e2d18bdda54a765e73f2c0b5be --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/diff.patch @@ -0,0 +1,19 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..bc04fba 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -135,12 +135,12 @@ load_checkpoint: ${agent.load_checkpoint} + # RGBD+mask+act history + # wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 +-# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json new file mode 100644 index 0000000000000000000000000000000000000000..293ed08a7657caa290bb6fb2588274bd3f75a5b5 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 45, 0.4444444444444444, 0.27840888504645445, 0.6185835599662125, 0.30912667516711145, 0.5990480035345346, 0.32985768470590726, 0.5639069154730775, 0.34584976735813877, 0.5483370497726087]]} \ No newline at end of file diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f5b059a86447f32077cbd3745930a89cda1b8cd4 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41d66fcf0a834c3bfb0b3b676bc76587709b7a60cac9e86c4f9dfc88334481c9 +size 1130489 diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a346a85d8234ae9f182e15894f774b0405f3d560 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27d2fca28dbf07e630def064d52cff1e4dec7c4ef45f7630a00451861ed77dc9 +size 1131474 diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f52b74ab16916d27ab82fbc6b0b4b8357e35cff7 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abcd97352dad09227ec188da612ffee0d94faf3dc73cc0fcc66c51924c302ee +size 1145346 diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..165c8fd866f77ebf34746d39585a8dd5e6183932 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12714e5cc65004ce4bac21af77d3fede418b9d7dd29150b35b5b30756951ad38 +size 1153288 diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e3835ee478e12436b3e65a9c9d359d457946124 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c9f5cbe3bf7ff9da1fdc97ea75d4e415b25f10928bed9131407f7f6624f53eb +size 1155936 diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..674e892847d6e510231378599423897db3de48eb --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log @@ -0,0 +1,273 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736290533.508863]: resetting environment +[INFO] [1736290533.511769]: cleared current plan +[INFO] [1736290533.512868]: moving to home +[INFO] [1736290534.514580]: reached home +[INFO] [1736290534.516018]: reset action history +[INFO] [1736290537.224771]: environment reset +Starting episode 0 +[INFO] [1736290537.226856]: resetting environment +[INFO] [1736290537.231855]: cleared current plan +[INFO] [1736290537.233042]: moving to home +[INFO] [1736290538.235572]: reached home +[INFO] [1736290538.237242]: reset action history +[INFO] [1736290540.945528]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736290548.627887]: resetting environment +[INFO] [1736290548.631110]: cleared current plan +[INFO] [1736290548.632125]: moving to home +[INFO] [1736290549.633357]: reached home +[INFO] [1736290549.634773]: reset action history +[INFO] [1736290552.341599]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/episode_rosbags/episode_0_2025-01-07-17-55-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2205299.log +started roslaunch server http://158.130.50.37:37559/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2205592] +process[print_text-2]: started with pid [2205593] +process[rosbag_record-3]: started with pid [2205594] +started bagging! +For topic gripper_width: timestamp difference is -109802324 for nearest: 1736290570778818529 - target: 1736290570888620853 at idx 243 +[INFO] [1736290579.364529]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736290580.098585]: Stored episode 1. +[INFO] [1736290580.099635]: Saving video... +[INFO] [1736290580.300668]: Video saved! +Starting episode 1 +[INFO] [1736290580.321017]: resetting environment +[INFO] [1736290580.331634]: cleared current plan +[INFO] [1736290580.332181]: moving to home +[INFO] [1736290584.034470]: reached home +[INFO] [1736290584.035397]: reset action history +[INFO] [1736290586.742053]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290591.992286]: resetting environment +[INFO] [1736290591.994756]: cleared current plan +[INFO] [1736290591.994904]: moving to home +[INFO] [1736290592.996451]: reached home +[INFO] [1736290592.997023]: reset action history +[INFO] [1736290595.703431]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/episode_rosbags/episode_1_2025-01-07-17-56-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2205299.log +started roslaunch server http://158.130.50.37:36101/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2205704] +process[print_text-5]: started with pid [2205705] +process[rosbag_record-6]: started with pid [2205729] +started bagging! +For topic gripper_width: timestamp difference is -122263586 for nearest: 1736290601078818644 - target: 1736290601201082230 at idx 78 +For topic gripper_width: timestamp difference is -120170351 for nearest: 1736290602078817133 - target: 1736290602198987484 at idx 87 +For topic gripper_width: timestamp difference is -116029125 for nearest: 1736290604078817982 - target: 1736290604194847107 at idx 111 +For topic gripper_width: timestamp difference is -107812408 for nearest: 1736290608078836676 - target: 1736290608186649084 at idx 163 +For topic gripper_width: timestamp difference is -104573941 for nearest: 1736290609678825402 - target: 1736290609783399343 at idx 184 +For topic gripper_width: timestamp difference is 104419070 for nearest: 1736290614278817731 - target: 1736290614174398661 at idx 244 +For topic gripper_width: timestamp difference is 101527470 for nearest: 1736290617478842038 - target: 1736290617377314568 at idx 249 +For topic gripper_width: timestamp difference is 104755833 for nearest: 1736290621478805258 - target: 1736290621374049425 at idx 249 +[INFO] [1736290622.641851]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736290623.470257]: Stored episode 2. +[INFO] [1736290623.470595]: Saving video... +[WARN] [1736290623.975175]: Plan exhausted +[WARN] [1736290624.013221]: Plan exhausted +[INFO] [1736290623.982412]: Video saved! +Starting episode 2 +[INFO] [1736290624.020639]: resetting environment +[INFO] [1736290624.040071]: cleared current plan +[INFO] [1736290624.040355]: moving to home +[INFO] [1736290627.742119]: reached home +[INFO] [1736290627.746351]: reset action history +[INFO] [1736290630.520679]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290645.044781]: resetting environment +[INFO] [1736290645.053112]: cleared current plan +[INFO] [1736290645.054615]: moving to home +[INFO] [1736290646.061157]: reached home +[INFO] [1736290646.061575]: reset action history +[INFO] [1736290648.820474]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/episode_rosbags/episode_2_2025-01-07-17-57-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2205299.log +started roslaunch server http://158.130.50.37:35385/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2205842] +process[print_text-8]: started with pid [2205843] +process[rosbag_record-9]: started with pid [2205867] +started bagging! +[INFO] [1736290675.762717]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736290676.564130]: Stored episode 3. +[INFO] [1736290676.566123]: Saving video... +[INFO] [1736290676.780365]: Video saved! +Starting episode 3 +[INFO] [1736290676.813980]: resetting environment +[INFO] [1736290676.835022]: cleared current plan +[INFO] [1736290676.838293]: moving to home +[INFO] [1736290680.552388]: reached home +[INFO] [1736290680.555411]: reset action history +[INFO] [1736290683.329890]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290685.853347]: resetting environment +[INFO] [1736290685.870815]: cleared current plan +[INFO] [1736290685.873483]: moving to home +[INFO] [1736290686.877956]: reached home +[INFO] [1736290686.885460]: reset action history +[INFO] [1736290689.674042]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/episode_rosbags/episode_3_2025-01-07-17-58-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2205299.log +started roslaunch server http://158.130.50.37:44221/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2206000] +process[print_text-11]: started with pid [2206001] +process[rosbag_record-12]: started with pid [2206002] +started bagging! +For topic gripper_width: timestamp difference is 107247524 for nearest: 1736290708245528393 - target: 1736290708138280869 at idx 241 +[INFO] [1736290716.822171]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit[print_text-11] killing on exit +[INFO] [1736290717.598642]: Stored episode 4. +[INFO] [1736290717.599216]: Saving video... +[INFO] [1736290718.084076]: Video saved! +Starting episode 4 +[INFO] [1736290718.122307]: resetting environment +[INFO] [1736290718.153870]: cleared current plan +[INFO] [1736290718.154087]: moving to home +[INFO] [1736290721.865292]: reached home +[INFO] [1736290721.869458]: reset action history +[INFO] [1736290724.658379]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290727.954763]: resetting environment +[INFO] [1736290727.964825]: cleared current plan +[INFO] [1736290727.965003]: moving to home +[INFO] [1736290728.967048]: reached home +[INFO] [1736290728.973601]: reset action history +[INFO] [1736290731.751514]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/episode_rosbags/episode_4_2025-01-07-17-58-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2205299.log +started roslaunch server http://158.130.50.37:38005/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2206111] +process[print_text-14]: started with pid [2206112] +process[rosbag_record-15]: started with pid [2206136] +started bagging! +For topic gripper_width: timestamp difference is -167044067 for nearest: 1736290746078817702 - target: 1736290746245861769 at idx 208 +For topic gripper_width: timestamp difference is 111526815 for nearest: 1736290752145484296 - target: 1736290752033957481 at idx 249 +[INFO] [1736290758.718859]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[INFO] [1736290759.551989]: Stored episode 5. +[INFO] [1736290759.555536]: Saving video... +[INFO] [1736290759.936320]: Video saved! +[INFO] [1736290759.966569]: resetting environment +[INFO] [1736290759.984282]: cleared current plan +[INFO] [1736290759.985271]: moving to home +[INFO] [1736290763.689247]: reached home +[INFO] [1736290763.699147]: reset action history +[INFO] [1736290766.554034]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/requirements.txt b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-metadata.json b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..504b1f6a67a1e8074bb8ca6e47cdb2b85cade31a --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T22:55:29.833044", + "startedAt": "2025-01-07T22:55:29.452951", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.324375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.25, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 789.7138137817383 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..696b13ef6762b45c6ea211015c72dbfd90baf3a9 --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "41d66fcf0a834c3bfb0b3b676bc76587709b7a60cac9e86c4f9dfc88334481c9", "size": 1130489, "path": "media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4"}, "global_step": 665, "_timestamp": 1736290773.2725763, "_runtime": 243.80552124977112, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "27d2fca28dbf07e630def064d52cff1e4dec7c4ef45f7630a00451861ed77dc9", "size": 1131474, "path": "media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4"}, "eval/num_success": 5.0, "episode": 45.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "2abcd97352dad09227ec188da612ffee0d94faf3dc73cc0fcc66c51924c302ee", "size": 1145346, "path": "media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "12714e5cc65004ce4bac21af77d3fede418b9d7dd29150b35b5b30756951ad38", "size": 1153288, "path": "media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "0c9f5cbe3bf7ff9da1fdc97ea75d4e415b25f10928bed9131407f7f6624f53eb", "size": 1155936, "path": "media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://tf9d2ohbjpqwy9y74sanfi3vcksegpvinb43p4z1j6y5lef2r5x0yf2lhd7lqpx0hzo8uvm3uc9v8vwfig4gw4d294vdetvae046z3vj8fwqbhzfew7sz21rdmf68mqu/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://8xbzfcgpv466rdie3lwqh1d57y8rmonq9x41shysfxsytaq8nh32a5p5fbx8eh3gu46vha61hu4ewlzgapxf79gn6cwusqw26e3uprhv547q88yzzoxbo8gfefjew16h:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 45, "total_success_rate": 0.4444444444444444, "total_success_rate_lb_0.01": 0.27840888504645445, "total_success_rate_ub_0.01": 0.6185835599662125, "total_success_rate_lb_0.025": 0.30912667516711145, "total_success_rate_ub_0.025": 0.5990480035345346, "total_success_rate_lb_0.05": 0.32985768470590726, "total_success_rate_ub_0.05": 0.5639069154730775, "total_success_rate_lb_0.1": 0.34584976735813877, "total_success_rate_ub_0.1": 0.5483370497726087, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "4ee8b1af4261e98151b69a138b648722a7afe53f2074d953442901ac57dbf6ef", "size": 524, "artifact_path": "wandb-client-artifact://ib396rr3c3acxhrfaa3isonxzjvbwt9c5xvlkl0s0exejvjxnp6rsl6fai0fy67z1a6iozgauxf68pje0f6lb6ewyac18taovlsoh4hjzx8ywcdd3w18278yysx0wb1s/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://975ywu4h7dhkq3o4wctrwfr41rm0j3jyjb9b8tq8dnsi8yktqg44bww3df0jkry47xem85kdlkakob6l1q4j5f8jbuqoy9ngn0y11d3aqx2k6rw4g2dgkx74g2vv1wwr:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 271}, "eval/total_success_rate": 0.4444444477558136, "eval/total_success_rate_lb_0.01": 0.27840888500213623, "eval/total_success_rate_ub_0.01": 0.6185835599899292, "eval/total_success_rate_lb_0.025": 0.30912667512893677, "eval/total_success_rate_ub_0.025": 0.5990480184555054, "eval/total_success_rate_lb_0.05": 0.3298576772212982, "eval/total_success_rate_ub_0.05": 0.5639069080352783, "eval/total_success_rate_lb_0.1": 0.34584978222846985, "eval/total_success_rate_ub_0.1": 0.5483370423316956} \ No newline at end of file diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug-internal.log b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..03d7cbbbdb5048822b5fc08421b72ca8c4f999fe --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug-internal.log @@ -0,0 +1,784 @@ +2025-01-07 17:55:29,467 INFO StreamThr :2205467 [internal.py:wandb_internal():85] W&B internal server running at pid: 2205467, started at: 2025-01-07 17:55:29.466724 +2025-01-07 17:55:29,468 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:55:29,479 INFO WriterThread:2205467 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/run-e3yh61zm.wandb +2025-01-07 17:55:29,480 DEBUG SenderThread:2205467 [sender.py:send():379] send: header +2025-01-07 17:55:29,487 DEBUG SenderThread:2205467 [sender.py:send():379] send: run +2025-01-07 17:55:29,751 INFO SenderThread:2205467 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files +2025-01-07 17:55:29,752 INFO SenderThread:2205467 [sender.py:_start_run_threads():1188] run started: e3yh61zm with start time 1736290529.467055 +2025-01-07 17:55:29,770 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:55:29,770 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:55:29,815 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:55:29,820 DEBUG HandlerThread:2205467 [system_info.py:__init__():26] System info init +2025-01-07 17:55:29,820 DEBUG HandlerThread:2205467 [system_info.py:__init__():41] System info init done +2025-01-07 17:55:29,820 INFO HandlerThread:2205467 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:55:29,820 INFO SystemMonitor:2205467 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:55:29,820 INFO HandlerThread:2205467 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:55:29,820 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:55:29,822 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:55:29,822 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:55:29,822 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:55:29,823 INFO SystemMonitor:2205467 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:55:29,832 DEBUG HandlerThread:2205467 [system_info.py:probe():152] Probing system +2025-01-07 17:55:29,836 DEBUG HandlerThread:2205467 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:55:29,842 DEBUG HandlerThread:2205467 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:55:29,842 DEBUG HandlerThread:2205467 [system_info.py:probe():200] Probing system done +2025-01-07 17:55:29,842 DEBUG HandlerThread:2205467 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:55:29.833044', 'startedAt': '2025-01-07T22:55:29.452951', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.324375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.25, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 789.7138137817383}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:55:29,843 INFO HandlerThread:2205467 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:55:29,843 INFO HandlerThread:2205467 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:55:29,843 DEBUG HandlerThread:2205467 [system_info.py:_save_code():44] Saving code +2025-01-07 17:55:29,848 DEBUG HandlerThread:2205467 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:55:29,848 DEBUG HandlerThread:2205467 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:55:29,910 DEBUG HandlerThread:2205467 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:55:29,910 INFO HandlerThread:2205467 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:55:29,914 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:55:29,914 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:55:29,914 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:55:29,915 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:55:30,158 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:55:30,158 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:55:30,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:55:30,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:55:30,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:30,162 INFO HandlerThread:2205467 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/tb" + save: true +} +control { + relay_id: "f58c3c22ca26407d829374d72f14e3ed" +} +_info { + stream_id: "e3yh61zm" +} + +2025-01-07 17:55:30,162 DEBUG HandlerThread:2205467 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:55:30,170 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/4zl12u4h-wandb-metadata.json +2025-01-07 17:55:30,188 INFO wandb-upload_1:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/khtzvqiv-code/FISH/eval_robot.py +2025-01-07 17:55:30,236 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/ovwg34yv-diff.patch +2025-01-07 17:55:30,247 DEBUG SenderThread:2205467 [sender.py:send():379] send: telemetry +2025-01-07 17:55:30,247 DEBUG SenderThread:2205467 [sender.py:send():379] send: tbrecord +2025-01-07 17:55:30,247 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:55:30,247 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 with policy live +2025-01-07 17:55:30,441 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/5faioorm-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:55:30,753 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/requirements.txt +2025-01-07 17:55:30,753 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code/FISH/eval_robot.py +2025-01-07 17:55:30,753 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-metadata.json +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/diff.patch +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code +2025-01-07 17:55:30,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code/FISH +2025-01-07 17:55:31,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:32,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:32,752 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:33,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:33,508 DEBUG SenderThread:2205467 [sender.py:send():379] send: config +2025-01-07 17:55:34,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:34,515 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:34,753 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:35,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:36,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:36,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:37,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:38,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:38,754 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:39,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:40,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:40,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:40,755 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:41,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:42,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:42,756 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:43,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:44,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:45,158 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:55:45,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:55:45,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:45,244 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:46,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:47,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:48,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:49,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:50,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:50,758 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:51,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:51,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:52,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:53,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:54,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:54,759 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:55:55,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:56,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:55:56,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:57,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:58,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:55:59,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:00,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:56:00,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:56:00,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:01,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:01,254 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:01,761 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml +2025-01-07 17:56:02,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:03,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:04,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:05,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:06,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:07,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:07,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:08,164 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:09,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:10,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:11,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:12,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:12,764 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:13,029 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:13,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:14,165 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:15,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:56:15,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:56:15,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:16,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:17,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:18,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:18,255 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:19,166 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:20,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:20,349 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:56:20,350 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:56:20,350 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 with policy now +2025-01-07 17:56:20,350 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:56:20,350 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:56:20,351 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:56:20,638 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/gm9moaxi-media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos +2025-01-07 17:56:20,766 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:56:20,767 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media +2025-01-07 17:56:21,045 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/wz1rsoth-media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 +2025-01-07 17:56:21,167 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:22,167 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:22,767 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:23,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:23,351 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:24,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:25,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:26,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:26,768 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:27,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:28,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:28,769 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:28,785 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:29,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:29,823 DEBUG SystemMonitor:2205467 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:56:29,824 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:56:30,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:56:30,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:56:30,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:31,168 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:32,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:32,770 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:56:33,063 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/bsfb9bdp-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:56:33,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:34,041 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:34,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:34,771 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:35,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:36,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:36,771 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:37,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:38,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:38,772 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:39,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:39,941 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:40,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:41,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:42,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:42,774 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:43,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:44,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:44,775 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:45,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:56:45,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:56:45,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:45,320 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:46,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:46,776 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:47,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:48,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:49,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:50,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:50,777 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:51,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:51,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:52,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:52,778 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:53,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:54,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:55,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:56,161 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:56:56,173 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:56,779 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:57,173 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:58,173 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:58,780 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:56:59,175 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:56:59,824 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:57:00,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:57:00,159 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:57:00,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:01,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:01,263 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:02,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:02,781 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:03,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:04,014 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:57:04,014 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 with policy now +2025-01-07 17:57:04,017 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:57:04,017 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:57:04,017 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:57:04,018 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:57:04,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:04,278 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/82pss7d7-media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 +2025-01-07 17:57:04,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:57:04,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:04,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 +2025-01-07 17:57:04,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:57:05,085 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/ojwi1mgs-media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 +2025-01-07 17:57:05,180 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:06,175 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:06,782 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:07,167 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:07,180 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:08,178 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:08,783 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:09,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:10,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:11,184 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:12,184 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:12,569 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:12,784 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:13,184 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:14,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:15,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:57:15,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:57:15,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:16,184 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:17,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:18,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:18,271 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:19,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:20,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:21,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:22,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:23,185 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:24,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:24,186 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:25,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:25,573 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:57:25,573 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:57:25,574 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:57:25,788 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:57:25,789 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:57:26,186 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:26,789 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:27,186 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:28,186 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:28,790 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:29,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:29,825 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:57:29,825 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:30,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:57:30,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:57:30,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:30,790 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:31,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:32,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:33,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:34,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:35,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:35,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:36,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:37,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:38,187 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:39,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:40,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:41,162 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:41,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:42,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:43,188 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:44,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:45,159 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:57:45,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:57:45,201 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:46,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:46,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:47,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:48,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:49,189 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:50,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:51,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:52,163 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:52,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:53,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:54,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:55,190 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:56,193 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:56,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:56,809 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:57:56,809 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 with policy now +2025-01-07 17:57:56,812 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:57:56,812 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:57:56,812 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:57:56,812 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:57:57,098 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/ft0q8hsn-media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 +2025-01-07 17:57:57,210 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:57,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:57:57,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 +2025-01-07 17:57:57,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:57:57,838 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:57:58,144 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/z9w04c9d-media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 +2025-01-07 17:57:58,206 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:58,799 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:57:59,204 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:57:59,825 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:58:00,176 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:58:00,176 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:58:00,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:01,208 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:02,208 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:02,801 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:03,216 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:03,331 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:04,213 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:04,801 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:05,213 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:06,215 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:06,403 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:58:06,403 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:58:06,404 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:58:06,802 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:58:06,802 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:58:06,802 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:07,074 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/peeu4rwv-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:58:07,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:08,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:08,803 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:08,929 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:09,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:10,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:10,803 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:11,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:12,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:12,804 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:13,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:14,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:14,217 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:15,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:58:15,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:58:15,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:16,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:17,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:18,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:19,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:19,262 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:20,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:21,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:22,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:23,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:24,218 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:25,171 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:25,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:26,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:27,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:28,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:29,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:29,826 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:58:30,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:58:30,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:58:30,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:30,259 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:30,811 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:31,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:32,219 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:33,220 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:34,220 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:35,220 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:36,172 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:36,220 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:37,222 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:38,118 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:58:38,118 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 with policy now +2025-01-07 17:58:38,118 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:58:38,119 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:58:38,119 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:58:38,121 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:58:38,221 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:38,387 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/2a2zlhgh-media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 +2025-01-07 17:58:38,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:58:38,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:38,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 +2025-01-07 17:58:38,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:58:39,099 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/49oiyhb6-media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 +2025-01-07 17:58:39,230 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:40,236 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:40,814 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:41,174 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:41,234 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:42,230 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:42,815 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:43,233 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:44,229 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:45,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:58:45,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:58:45,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:46,234 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:46,264 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:46,816 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:47,242 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:48,236 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:48,455 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:58:48,456 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:58:48,457 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:58:48,817 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:58:48,817 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:58:48,818 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:49,236 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:50,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:50,818 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:51,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:51,752 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:52,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:52,819 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:53,237 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:54,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:54,820 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:58:55,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:56,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:57,176 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:58:57,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:58,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:59,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:58:59,826 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:59:00,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:59:00,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:59:00,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:01,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:02,238 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:02,268 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:03,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:04,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:05,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:06,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:07,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:07,381 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:08,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:08,825 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:09,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:10,239 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:11,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:12,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:13,194 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:13,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:14,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:14,827 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:15,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:59:15,160 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:59:15,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:16,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:17,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:18,240 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:18,241 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:19,248 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:19,965 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:59:19,965 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 with policy now +2025-01-07 17:59:19,966 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:59:19,966 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:59:19,966 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:19,968 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:20,230 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/bdve6x6w-media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 +2025-01-07 17:59:20,243 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:20,829 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:59:20,830 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:20,830 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 +2025-01-07 17:59:20,830 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval +2025-01-07 17:59:21,150 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/pib94gwe-media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 +2025-01-07 17:59:21,244 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:22,244 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:23,251 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:23,696 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:24,250 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:24,831 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:25,250 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:26,248 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:27,318 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:28,296 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:28,832 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:29,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:29,597 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:29,826 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 17:59:30,169 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:59:30,169 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:59:30,306 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:31,305 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:32,304 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:32,513 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:59:32,514 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:32,514 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:32,811 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:59:32,811 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:59:32,834 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:59:32,834 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:59:32,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:59:32,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media +2025-01-07 17:59:32,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval +2025-01-07 17:59:32,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table +2025-01-07 17:59:32,949 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:59:33,049 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/x3wsjfcs-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:59:33,094 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/hnlwicwb-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:59:33,296 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:33,326 INFO wandb-upload_1:2205467 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4lzqx5y_ +2025-01-07 17:59:33,370 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:59:33,413 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:59:33,695 INFO SenderThread:2205467 [sender.py:send_request_log_artifact():1525] logged artifact run-e3yh61zm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAzMTk4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTczMTgzNA==', 'latestArtifact': None}} +2025-01-07 17:59:33,695 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:59:33,695 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 17:59:33,696 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:59:33,696 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,696 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,696 DEBUG SenderThread:2205467 [sender.py:send():379] send: telemetry +2025-01-07 17:59:33,697 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,697 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,697 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,698 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,698 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,700 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,700 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,700 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,701 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,701 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,701 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,701 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,701 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,702 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,702 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,702 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,702 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,703 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,703 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,703 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,703 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:33,704 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:33,704 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:59:33,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 17:59:33,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:59:33,835 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:59:33,836 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval +2025-01-07 17:59:33,955 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/tgj5o8az-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:59:34,029 INFO wandb-upload_1:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/uyc9lbf6-media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:59:34,076 INFO wandb-upload_0:2205467 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0ldto3x9 +2025-01-07 17:59:34,296 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:34,535 INFO SenderThread:2205467 [sender.py:send_request_log_artifact():1525] logged artifact run-e3yh61zm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAzMjAxMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTczMTgzNg==', 'latestArtifact': None}} +2025-01-07 17:59:34,535 DEBUG SenderThread:2205467 [sender.py:send():379] send: files +2025-01-07 17:59:34,535 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json with policy now +2025-01-07 17:59:34,536 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 17:59:34,536 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:59:34,536 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:59:34,810 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/fmwovnj0-media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 17:59:34,836 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 17:59:34,836 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 17:59:35,176 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:35,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:36,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:37,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:38,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:39,297 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:40,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:41,176 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:41,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:42,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:43,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:44,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:45,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:59:45,161 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:59:45,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:46,288 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:46,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:47,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:48,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:49,298 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:50,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:51,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:52,180 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:52,304 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:52,843 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml +2025-01-07 17:59:53,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:54,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:55,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:56,299 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:57,300 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:58,177 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:59:58,300 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:59,300 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:59:59,827 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 18:00:00,160 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:00:00,161 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:00:00,300 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:00,873 DEBUG SenderThread:2205467 [sender.py:send():379] send: exit +2025-01-07 18:00:00,873 INFO SenderThread:2205467 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:send_exit():588] handling runtime: 271 +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:send_exit():594] send defer +2025-01-07 18:00:00,874 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,874 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:00:00,874 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:00:00,874 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:00:00,875 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:00:00,875 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:00,875 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:00:00,875 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:00:00,875 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:00:00,875 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:00:00,876 DEBUG SystemMonitor:2205467 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:00:00,876 DEBUG SystemMonitor:2205467 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:00:00,926 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:00:00,926 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:00:00,926 INFO HandlerThread:2205467 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:00:00,927 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:00,927 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:00:00,927 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:00:00,927 DEBUG SenderThread:2205467 [sender.py:send():379] send: stats +2025-01-07 18:00:00,927 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,927 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:00:00,927 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:00,927 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:00:00,927 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:00:00,928 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:00,928 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:00:01,846 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 18:00:02,847 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 18:00:07,522 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:07,522 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:07,522 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:00:07,523 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:07,523 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:00:07,524 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:00:07,524 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:07,524 DEBUG SenderThread:2205467 [sender.py:send():379] send: history +2025-01-07 18:00:07,524 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:00:07,524 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:07,524 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:00:07,525 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:00:07,525 DEBUG SenderThread:2205467 [sender.py:send():379] send: summary +2025-01-07 18:00:07,529 INFO SenderThread:2205467 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:00:07,529 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:07,529 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:00:07,529 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:00:07,529 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:07,529 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:00:07,530 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:07,530 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:00:07,530 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:00:07,530 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:07,530 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:07,530 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:00:07,530 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:07,530 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:00:07,849 INFO Thread-12 :2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 18:00:07,875 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:10,151 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:00:10,151 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:10,151 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:10,152 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:00:10,152 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:10,152 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:00:10,152 INFO SenderThread:2205467 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:00:10,154 INFO SenderThread:2205467 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:00:10,156 INFO SenderThread:2205467 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:00:10,170 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:00:10,170 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:10,170 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:00:10,170 DEBUG SenderThread:2205467 [sender.py:send():379] send: artifact +2025-01-07 18:00:10,384 INFO SenderThread:2205467 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:00:10,384 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:10,384 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:00:10,384 INFO SenderThread:2205467 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml config.yaml +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json wandb-summary.json +2025-01-07 18:00:10,850 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/requirements.txt requirements.txt +2025-01-07 18:00:10,851 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:00:10,851 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 18:00:10,851 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log output.log +2025-01-07 18:00:10,851 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/diff.patch diff.patch +2025-01-07 18:00:10,852 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 media/videos/eval/4_eval_4_0c9f5cbe3bf7ff9da1fd.mp4 +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 media/videos/eval/1_eval_1_27d2fca28dbf07e630de.mp4 +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 media/videos/eval/3_eval_3_12714e5cc65004ce4bac.mp4 +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 media/videos/eval/2_eval_2_2abcd97352dad09227ec.mp4 +2025-01-07 18:00:10,856 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 media/videos/eval/0_eval_0_41d66fcf0a834c3bfb0b.mp4 +2025-01-07 18:00:10,859 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json media/table/eval/total_success_rate_ci_6_4ee8b1af4261e98151b6.table.json +2025-01-07 18:00:10,859 INFO SenderThread:2205467 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:00:10,859 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:00:10,859 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:10,859 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:00:10,860 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:10,860 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:00:10,860 INFO SenderThread:2205467 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:00:10,876 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:10,876 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:11,035 INFO wandb-upload_2:2205467 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/wandb-summary.json +2025-01-07 18:00:11,046 INFO wandb-upload_0:2205467 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/config.yaml +2025-01-07 18:00:11,077 INFO wandb-upload_1:2205467 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/requirements.txt +2025-01-07 18:00:11,093 INFO wandb-upload_4:2205467 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/files/output.log +2025-01-07 18:00:11,181 INFO wandb-upload_3:2205467 [upload_job.py:push():130] Uploaded file /tmp/tmpfpvzt6rqwandb/f4obak7k-events.out.tfevents.1736290530.leonmkim-ROG-Strix-G15CS-G15CS.2205299.0 +2025-01-07 18:00:11,381 INFO Thread-11 (_thread_body):2205467 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:00:11,381 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:11,381 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:00:11,381 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:11,382 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:00:11,382 INFO SenderThread:2205467 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:00:11,383 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:00:11,383 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:11,383 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:00:11,383 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:11,383 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:00:11,383 INFO SenderThread:2205467 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:00:11,649 INFO SenderThread:2205467 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:00:11,649 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:00:11,649 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:11,649 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:00:11,649 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:11,649 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:00:11,650 INFO SenderThread:2205467 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:00:11,650 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:00:11,650 INFO HandlerThread:2205467 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:00:11,650 DEBUG SenderThread:2205467 [sender.py:send():379] send: final +2025-01-07 18:00:11,650 DEBUG SenderThread:2205467 [sender.py:send():379] send: footer +2025-01-07 18:00:11,650 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: defer +2025-01-07 18:00:11,650 INFO SenderThread:2205467 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:00:11,651 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:11,651 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:11,651 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:00:11,651 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:00:11,651 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:00:11,653 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:00:11,653 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:00:11,654 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:11,717 DEBUG SenderThread:2205467 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:00:11,717 INFO MainThread:2205467 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:00:11,717 INFO MainThread:2205467 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:00:11,718 INFO MainThread:2205467 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:00:11,718 DEBUG HandlerThread:2205467 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:00:11,718 INFO HandlerThread:2205467 [handler.py:finish():882] shutting down handler +2025-01-07 18:00:12,653 INFO WriterThread:2205467 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/run-e3yh61zm.wandb +2025-01-07 18:00:12,717 INFO SenderThread:2205467 [sender.py:finish():1615] shutting down sender +2025-01-07 18:00:12,717 INFO SenderThread:2205467 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:00:12,717 INFO SenderThread:2205467 [file_pusher.py:join():175] waiting for file pusher diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug.log b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..61b24a4763edba4efec4f633bd283f768429181c --- /dev/null +++ b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Configure stats pid to 2205299 +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/settings +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:55:29,459 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug.log +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/logs/debug-internal.log +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm'} +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:init():619] starting backend +2025-01-07 17:55:29,460 INFO MainThread:2205299 [wandb_init.py:init():623] setting up manager +2025-01-07 17:55:29,465 INFO MainThread:2205299 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:55:29,466 INFO MainThread:2205299 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:55:29,479 INFO MainThread:2205299 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:55:29,485 INFO MainThread:2205299 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:55:29,770 INFO MainThread:2205299 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:55:29,810 INFO MainThread:2205299 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:55:29,811 INFO MainThread:2205299 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:55:30,158 INFO MainThread:2205299 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:55:30,159 INFO MainThread:2205299 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:55:30,159 INFO MainThread:2205299 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:55:30,159 INFO MainThread:2205299 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:55:30,161 INFO MainThread:2205299 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:55:30,162 INFO MainThread:2205299 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/e3yh61zm/tb, True +2025-01-07 17:55:33,507 INFO MainThread:2205299 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:00:12,719 WARNING MsgRouterThr:2205299 [router.py:message_loop():77] message_loop has been closed diff --git a/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/run-e3yh61zm.wandb b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/run-e3yh61zm.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e392b43eb3a46636b63fd524ac5e4b979aabb173 Binary files /dev/null and b/e3yh61zm/wandb/run-20250107_175529-e3yh61zm/run-e3yh61zm.wandb differ diff --git a/ft0xy3w8/.hydra/config.yaml b/ft0xy3w8/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/ft0xy3w8/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ft0xy3w8/.hydra/hydra.yaml b/ft0xy3w8/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52ae76bfb296442ecb8870d557a058050b56498e --- /dev/null +++ b/ft0xy3w8/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/ft0xy3w8 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ft0xy3w8/.hydra/overrides.yaml b/ft0xy3w8/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ft0xy3w8/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ft0xy3w8/eval_policy.log b/ft0xy3w8/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..f1030e3d632de32a03506052f6023dfb33cb1d55 --- /dev/null +++ b/ft0xy3w8/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 20:31:29,617][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:31:29,620][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 20:31:31,032][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/snapshot_11749.pt b/snapshot_11749.pt new file mode 100644 index 0000000000000000000000000000000000000000..05d2b39a1ce59122fbccc08142120370218e53ec --- /dev/null +++ b/snapshot_11749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:969b7ec30923b9788a9ed4d9fe6f0a72618f2b15329b82dca4ebf9105e854bfa +size 910102870 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e08124d10b73cc89e71209ab661756f862d07e64 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68cecb68434c3093cd515d4513487006291c2a2f64b9995128a7428060412c63 +size 910102870 diff --git a/snapshot_13336.pt b/snapshot_13336.pt new file mode 100644 index 0000000000000000000000000000000000000000..360ff4a19448d942b9dbe8dae0accdbbe029aa18 --- /dev/null +++ b/snapshot_13336.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f113162e356ed6fa8f37fecd44d2296bb1e3c5ec2ce22ec8fd6839912c8181f +size 910102870 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..00f264c10d1310681ae59a957c6ae8e700980dcd --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac6dd4f1a8d54ee869d8ca6566741bb86d7aacc2e16a1d63600f893b3a75c783 +size 910102870 diff --git a/snapshot_14249.pt b/snapshot_14249.pt new file mode 100644 index 0000000000000000000000000000000000000000..32fba14a8ccbb5195124e33cd27968b52d9007e6 --- /dev/null +++ b/snapshot_14249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e672eab9f0e46fc23fedbcf995cf7caa271a5b866d3d5d5acd2c29a5346547ff +size 910102870 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfbc7e015c1cdf5c2d07c34f8d6440dddd22534b --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca37d69008d3b0f3ea788f1e25b2260729ed5a46050f6e7c8a85d338921aa9a +size 910102870 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d0cb2bfc1244f69c5ca09e3563570a63ed6d650 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd520af42ef0aeb8af21458a9c9838b1cff584eeb3530511d63d54cae3dad10 +size 910102360 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c97c78b459c5abbc2cfa32d2464f38a92a45ccd0 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29494b448013120dd37e48f746d123c73d91a331cdd369e0a084753f6e5c0ca2 +size 910102360 diff --git a/snapshot_6749.pt b/snapshot_6749.pt new file mode 100644 index 0000000000000000000000000000000000000000..2419c0584ddaacdd4f9568553d8482373b84c371 --- /dev/null +++ b/snapshot_6749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:697f727aff98e4a814787d1bb92f8e9e27cdf49bd3db3620703bc452a2c8e355 +size 910102360 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..b28b2fe66edecff9e842d1e2300bb69f8d5b9eb9 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb9aec544728cbf5128fafb9103c53f5db411f1fb5431428ee33b099404eb916 +size 910102360 diff --git a/snapshot_8999.pt b/snapshot_8999.pt new file mode 100644 index 0000000000000000000000000000000000000000..42e0f14bcf22e2b26a4e6214e58a6a0f70c68dca --- /dev/null +++ b/snapshot_8999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04a4d240b5ea55d6d278ea03aff5a89b29d940c7d1fa67b77b33cca73050a0e6 +size 910102360 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfbc7e015c1cdf5c2d07c34f8d6440dddd22534b --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca37d69008d3b0f3ea788f1e25b2260729ed5a46050f6e7c8a85d338921aa9a +size 910102870 diff --git a/vd7nl959/.hydra/config.yaml b/vd7nl959/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/vd7nl959/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/vd7nl959/.hydra/hydra.yaml b/vd7nl959/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d4a33d772eadaca3fcae3a6996cdba83e087a51 --- /dev/null +++ b/vd7nl959/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/vd7nl959/.hydra/overrides.yaml b/vd7nl959/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/vd7nl959/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/vd7nl959/episode_rosbags/aligned_depth_to_color_K.npy b/vd7nl959/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/vd7nl959/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/vd7nl959/episode_rosbags/cam_tf_world.npy b/vd7nl959/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/vd7nl959/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/vd7nl959/episode_rosbags/color_K.npy b/vd7nl959/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/vd7nl959/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/vd7nl959/episode_rosbags/depth_K.npy b/vd7nl959/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/vd7nl959/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/vd7nl959/eval_robot.log b/vd7nl959/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..747f7b469e4a946b377387eb55abe0d6ecc85126 --- /dev/null +++ b/vd7nl959/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 20:35:51,675][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:35:51,678][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/vd7nl959/eval_video/0_eval.mp4 b/vd7nl959/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2527dd5ece45c104983ede3721a9441dac646550 --- /dev/null +++ b/vd7nl959/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb31d93c1da1119a0f99dead6e8de7cc9426ba19ba355cdda44cdd7ec53264b +size 1142369 diff --git a/vd7nl959/eval_video/1_eval.mp4 b/vd7nl959/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4dab3df107df2b9074d85fc54b5f69fdd04ac13e --- /dev/null +++ b/vd7nl959/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04da90b434a68619a5e33317b744b174f46337bf762360953ceab2e0bccc14ed +size 1142628 diff --git a/vd7nl959/eval_video/2_eval.mp4 b/vd7nl959/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67e4792cbed72fe4bd73f72e0a30288e45c5b4e1 --- /dev/null +++ b/vd7nl959/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf3e651ced546db728b0637e20b6faa1b21ce41d2776b443e9772929b3e1146c +size 1149554 diff --git a/vd7nl959/eval_video/3_eval.mp4 b/vd7nl959/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aba4c4aa0a3d05bfbccd5dbf133e5d5606bb5947 --- /dev/null +++ b/vd7nl959/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a4bd86fb7d6eb3cf5e41f6b667df345e5062e1c80bf8233f805f14559c1698d +size 1151827 diff --git a/vd7nl959/eval_video/4_eval.mp4 b/vd7nl959/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..910e643a42675f6a17c286a79a48195e363f2930 --- /dev/null +++ b/vd7nl959/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7007aef1a567b8508ad9494ca8afb542555722db42b465344bd8015a29ef78bd +size 1148908 diff --git a/vd7nl959/tb/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 b/vd7nl959/tb/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 new file mode 100644 index 0000000000000000000000000000000000000000..ce3ebdeb3b5b8d553265a8d834c0d52c8a0e99fc --- /dev/null +++ b/vd7nl959/tb/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d420998d1b8f3b4f3cffe2cf9ce58cecb69146d7cb6d6211576de8f04495320 +size 6539 diff --git a/vd7nl959/wandb/debug-internal.log b/vd7nl959/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0c77048b4f24d32be1c4af49c0a8b5a3388b0315 --- /dev/null +++ b/vd7nl959/wandb/debug-internal.log @@ -0,0 +1,753 @@ +2025-01-06 20:35:57,446 INFO StreamThr :2112019 [internal.py:wandb_internal():85] W&B internal server running at pid: 2112019, started at: 2025-01-06 20:35:57.445835 +2025-01-06 20:35:57,447 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:35:57,458 INFO WriterThread:2112019 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/run-vd7nl959.wandb +2025-01-06 20:35:57,459 DEBUG SenderThread:2112019 [sender.py:send():379] send: header +2025-01-06 20:35:57,466 DEBUG SenderThread:2112019 [sender.py:send():379] send: run +2025-01-06 20:35:57,702 INFO SenderThread:2112019 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files +2025-01-06 20:35:57,702 INFO SenderThread:2112019 [sender.py:_start_run_threads():1188] run started: vd7nl959 with start time 1736213757.4465 +2025-01-06 20:35:57,720 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:35:57,720 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:35:57,805 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:35:57,810 DEBUG HandlerThread:2112019 [system_info.py:__init__():26] System info init +2025-01-06 20:35:57,810 DEBUG HandlerThread:2112019 [system_info.py:__init__():41] System info init done +2025-01-06 20:35:57,810 INFO HandlerThread:2112019 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:35:57,810 INFO SystemMonitor:2112019 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:35:57,810 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:35:57,811 INFO HandlerThread:2112019 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:35:57,812 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:35:57,812 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:35:57,813 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:35:57,813 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:35:57,823 DEBUG HandlerThread:2112019 [system_info.py:probe():152] Probing system +2025-01-06 20:35:57,824 DEBUG HandlerThread:2112019 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:35:57,829 DEBUG HandlerThread:2112019 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:35:57,829 DEBUG HandlerThread:2112019 [system_info.py:probe():200] Probing system done +2025-01-06 20:35:57,829 DEBUG HandlerThread:2112019 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:35:57.823325', 'startedAt': '2025-01-07T01:35:57.437487', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.21275, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.98, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.61, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 505.5578384399414}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:35:57,829 INFO HandlerThread:2112019 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:35:57,829 INFO HandlerThread:2112019 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:35:57,829 DEBUG HandlerThread:2112019 [system_info.py:_save_code():44] Saving code +2025-01-06 20:35:57,831 DEBUG HandlerThread:2112019 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:35:57,831 DEBUG HandlerThread:2112019 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:35:57,915 DEBUG HandlerThread:2112019 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:35:57,915 INFO HandlerThread:2112019 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:35:57,918 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:35:57,918 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:35:57,919 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:35:57,919 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:35:58,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:35:58,159 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:35:58,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:58,161 DEBUG SenderThread:2112019 [sender.py:send():379] send: telemetry +2025-01-06 20:35:58,181 INFO wandb-upload_2:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/yfojxhab-diff.patch +2025-01-06 20:35:58,181 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/deoj5bdl-wandb-metadata.json +2025-01-06 20:35:58,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:35:58,201 INFO HandlerThread:2112019 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/tb" + save: true +} +control { + relay_id: "deba67f5c4a4425e96a5a9a7e46a27bb" +} +_info { + stream_id: "vd7nl959" +} + +2025-01-06 20:35:58,201 DEBUG HandlerThread:2112019 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:35:58,202 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:35:58,218 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/433434gp-code/FISH/eval_robot.py +2025-01-06 20:35:58,275 DEBUG SenderThread:2112019 [sender.py:send():379] send: tbrecord +2025-01-06 20:35:58,276 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:35:58,276 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 with policy live +2025-01-06 20:35:58,513 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/e2xw3ogu-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:35:58,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:35:58,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-metadata.json +2025-01-06 20:35:58,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code/FISH/eval_robot.py +2025-01-06 20:35:58,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/diff.patch +2025-01-06 20:35:58,705 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/requirements.txt +2025-01-06 20:35:58,705 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:35:58,705 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code +2025-01-06 20:35:58,705 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code/FISH +2025-01-06 20:35:59,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:00,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:00,703 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:01,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:02,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:02,209 DEBUG SenderThread:2112019 [sender.py:send():379] send: config +2025-01-06 20:36:03,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:03,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:04,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:04,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:05,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:06,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:07,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:08,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:08,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:08,706 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:09,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:10,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:10,706 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:11,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:12,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:13,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:36:13,159 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:36:13,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:13,295 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:14,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:15,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:16,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:17,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:18,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:19,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:19,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:20,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:21,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:22,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:23,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:24,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:24,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:25,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:26,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:26,710 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:27,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:28,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:36:28,159 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:36:28,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:28,711 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:29,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:29,934 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:30,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:30,712 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml +2025-01-06 20:36:30,712 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:31,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:32,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:33,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:34,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:35,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:35,164 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:36,164 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:37,164 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:38,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:39,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:40,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:41,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:41,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:42,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:43,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:36:43,159 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:36:43,205 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:44,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:45,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:46,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:46,310 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:47,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:48,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:49,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:50,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:51,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:52,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:52,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:53,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:54,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:55,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:56,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:56,583 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:36:56,583 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 with policy now +2025-01-06 20:36:56,583 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:36:56,584 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:36:56,584 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:36:56,584 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:36:56,848 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/pwh2lllx-media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 +2025-01-06 20:36:57,114 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/be2qygjy-media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 +2025-01-06 20:36:57,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:57,590 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:57,813 DEBUG SystemMonitor:2112019 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:36:57,814 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:36:58,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:36:58,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:36:58,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:58,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:59,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:00,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:01,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:02,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:02,721 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:03,002 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:03,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:04,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:04,722 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:05,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:05,722 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:37:05,974 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/1czzhzeo-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:37:06,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:06,722 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:07,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:08,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:08,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:08,723 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:09,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:10,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:10,724 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:11,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:12,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:13,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:37:13,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:37:13,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:13,278 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:14,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:15,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:16,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:17,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:18,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:19,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:19,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:20,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:21,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:22,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:23,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:24,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:24,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:25,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:26,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:27,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:27,814 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:37:28,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:37:28,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:37:28,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:29,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:29,324 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:30,173 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:31,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:32,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:33,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:34,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:35,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:35,175 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:36,175 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:36,732 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:36,755 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:37:36,755 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 with policy now +2025-01-06 20:37:36,756 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:37:36,768 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:37:36,769 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:37:36,769 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:37:37,005 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/jiyvyyvk-media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 +2025-01-06 20:37:37,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:37,732 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:37:37,732 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 +2025-01-06 20:37:37,733 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:37:38,024 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/1749qcx4-media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 +2025-01-06 20:37:38,178 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:38,733 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:39,180 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:40,178 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:40,180 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:41,181 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:42,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:42,734 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:43,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:37:43,166 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:37:43,209 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:44,181 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:44,735 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:45,181 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:45,421 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:46,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:47,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:48,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:49,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:50,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:51,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:51,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:52,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:53,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:54,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:55,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:56,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:57,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:57,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:57,739 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:37:57,815 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:37:58,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:37:58,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:37:58,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:58,660 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:37:58,660 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:37:58,661 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:37:58,739 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:58,740 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:37:59,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:00,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:00,741 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:01,189 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:02,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:02,579 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:02,741 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:03,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:04,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:04,742 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:05,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:06,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:07,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:07,978 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:08,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:09,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:10,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:11,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:12,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:13,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:38:13,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:38:13,205 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:13,251 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:14,186 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:15,186 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:16,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:17,186 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:18,186 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:19,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:19,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:20,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:21,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:22,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:23,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:24,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:24,381 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:24,749 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:25,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:26,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:27,188 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:27,815 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:38:28,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:38:28,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:38:28,205 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:29,190 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:29,639 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:29,681 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:38:29,682 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:38:29,682 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 with policy now +2025-01-06 20:38:29,682 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:38:29,682 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:38:29,683 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:38:29,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:38:29,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 +2025-01-06 20:38:29,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:38:29,944 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/3lx7aqdl-media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 +2025-01-06 20:38:30,194 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:30,214 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/00un4lro-media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 +2025-01-06 20:38:30,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:31,197 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:31,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:32,202 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:33,210 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:34,204 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:34,752 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:35,203 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:35,541 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:36,204 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:37,207 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:38,207 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:38,754 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:39,208 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:40,208 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:41,136 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:41,206 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:41,452 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:38:41,452 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:38:41,453 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:38:41,755 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:38:41,755 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:38:42,007 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/yyhpk17r-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:38:42,229 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:42,755 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:43,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:38:43,165 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:38:43,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:44,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:44,756 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:45,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:46,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:46,574 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:46,757 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:47,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:48,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:49,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:50,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:51,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:52,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:52,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:53,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:54,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:55,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:56,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:57,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:57,816 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:38:57,816 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:58,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:38:58,161 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:38:58,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:59,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:00,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:01,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:02,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:03,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:03,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:04,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:05,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:06,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:06,763 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:07,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:08,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:08,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:08,764 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:09,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:10,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:11,217 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:12,216 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:13,164 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:39:13,164 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:39:13,221 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:13,324 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:13,629 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:39:13,629 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:39:13,630 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 with policy now +2025-01-06 20:39:13,630 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:39:13,630 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:39:13,630 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:39:13,766 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:39:13,766 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 +2025-01-06 20:39:13,766 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:39:13,908 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/1vzzwz6d-media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 +2025-01-06 20:39:14,165 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/8pvkcqip-media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 +2025-01-06 20:39:14,243 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:14,766 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:15,224 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:16,224 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:17,237 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:18,237 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:18,768 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:19,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:19,240 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:20,225 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:21,230 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:22,231 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:22,506 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:39:22,506 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:39:22,506 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:39:22,769 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:39:22,769 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:22,769 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:39:23,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:24,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:24,770 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:25,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:25,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:26,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:26,770 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:27,231 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:27,816 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:39:28,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:39:28,161 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:39:28,231 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:29,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:30,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:30,276 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:31,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:32,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:33,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:34,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:35,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:36,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:36,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:37,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:38,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:39,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:40,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:41,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:41,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:42,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:42,776 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:43,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:39:43,161 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:39:43,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:44,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:45,234 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:46,234 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:46,777 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:47,073 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:47,234 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:48,234 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:49,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:50,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:51,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:52,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:52,449 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:52,780 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:53,243 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:53,883 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:39:53,883 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 with policy now +2025-01-06 20:39:53,885 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:39:53,886 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:39:53,886 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:39:53,887 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:39:54,161 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/1dcty9no-media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 +2025-01-06 20:39:54,236 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:54,781 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:54,781 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:39:54,781 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 +2025-01-06 20:39:54,781 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:39:55,088 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/ew2wcu23-media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 +2025-01-06 20:39:55,248 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:56,245 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:57,244 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:57,718 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:57,817 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:39:58,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:39:58,168 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:39:58,254 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:58,782 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:59,266 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:00,257 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:01,254 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:02,254 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:02,555 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:40:02,555 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:02,555 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:02,783 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:02,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:02,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:02,784 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:40:02,784 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:40:02,978 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:40:03,029 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/ilxnxfmq-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:03,260 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:03,275 INFO wandb-upload_0:2112019 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn4nq66mv +2025-01-06 20:40:03,366 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:40:03,375 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:40:03,675 INFO SenderThread:2112019 [sender.py:send_request_log_artifact():1525] logged artifact run-vd7nl959-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMwMjk3Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYzNzE0OQ==', 'latestArtifact': None}} +2025-01-06 20:40:03,675 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:40:03,675 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:40:03,676 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:40:03,676 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,676 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:03,676 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,677 DEBUG SenderThread:2112019 [sender.py:send():379] send: telemetry +2025-01-06 20:40:03,677 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,680 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,680 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,681 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,681 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,681 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,682 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,682 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,682 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,682 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,683 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,683 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,683 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,683 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,684 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,684 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,684 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,684 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,684 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,685 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,685 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,685 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,685 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media +2025-01-06 20:40:03,785 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table +2025-01-06 20:40:03,785 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval +2025-01-06 20:40:03,867 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/2ne47f8i-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:40:04,008 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/6bd135am-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2025-01-06 20:40:04,084 INFO wandb-upload_2:2112019 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4imv0blj +2025-01-06 20:40:04,110 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/5f4vy0vc-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:40:04,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:04,515 INFO SenderThread:2112019 [sender.py:send_request_log_artifact():1525] logged artifact run-vd7nl959-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMwMjk5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYzNzE1MA==', 'latestArtifact': None}} +2025-01-06 20:40:04,515 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:40:04,515 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json with policy now +2025-01-06 20:40:04,516 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:40:04,516 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:04,516 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:04,781 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/rndff0hh-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2025-01-06 20:40:04,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:04,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:05,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:06,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:07,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:08,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:09,176 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:09,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:10,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:11,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:12,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:13,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:40:13,162 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:40:13,256 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:13,609 DEBUG SenderThread:2112019 [sender.py:send():379] send: exit +2025-01-06 20:40:13,610 INFO SenderThread:2112019 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:40:13,610 INFO SenderThread:2112019 [sender.py:send_exit():588] handling runtime: 255 +2025-01-06 20:40:13,611 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:13,611 INFO SenderThread:2112019 [sender.py:send_exit():594] send defer +2025-01-06 20:40:13,611 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,611 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:40:13,611 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:13,611 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:40:13,611 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:40:13,611 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,611 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:40:13,612 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:13,612 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:40:13,612 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:40:13,612 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,612 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:40:13,612 INFO HandlerThread:2112019 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:40:13,612 DEBUG SystemMonitor:2112019 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:40:13,612 DEBUG SystemMonitor:2112019 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:40:13,613 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:40:13,613 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:40:13,664 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:40:13,664 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:40:13,664 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:40:13,665 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:13,665 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:40:13,665 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:40:13,665 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:40:13,665 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,665 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:40:13,665 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:13,665 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:40:13,665 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:40:13,666 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,666 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:40:13,788 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:14,789 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:20,562 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:20,562 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:20,562 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:40:20,562 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:20,563 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:20,563 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:40:20,563 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:40:20,563 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:20,563 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:40:20,564 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:20,564 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:20,564 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:40:20,565 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:20,565 DEBUG SenderThread:2112019 [sender.py:send():379] send: summary +2025-01-06 20:40:20,565 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:20,565 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:20,565 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:40:20,565 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:40:20,565 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:20,566 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:40:20,566 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:20,566 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:40:20,566 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:40:20,566 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:20,566 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:20,566 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:40:20,566 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:20,566 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:40:20,611 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:20,791 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml +2025-01-06 20:40:20,791 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:23,793 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:40:23,793 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:23,793 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:23,794 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:40:23,794 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:23,794 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:40:23,794 INFO SenderThread:2112019 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:40:23,796 INFO SenderThread:2112019 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:40:23,798 INFO SenderThread:2112019 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:40:23,813 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:40:23,813 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:23,813 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:40:23,813 DEBUG SenderThread:2112019 [sender.py:send():379] send: artifact +2025-01-06 20:40:24,101 INFO SenderThread:2112019 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:40:24,101 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:24,101 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:40:24,101 INFO SenderThread:2112019 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:40:24,612 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:24,792 INFO SenderThread:2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:24,792 INFO SenderThread:2112019 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml config.yaml +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json wandb-summary.json +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/requirements.txt requirements.txt +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log output.log +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/diff.patch diff.patch +2025-01-06 20:40:24,795 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:40:24,798 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 +2025-01-06 20:40:24,798 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:40:24,799 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:24,799 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:24,799 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:40:24,801 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:24,801 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:40:24,801 INFO SenderThread:2112019 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:40:24,988 INFO wandb-upload_2:2112019 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/requirements.txt +2025-01-06 20:40:24,999 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:25,010 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml +2025-01-06 20:40:25,041 INFO wandb-upload_4:2112019 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:25,086 INFO wandb-upload_3:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/holtclnn-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:25,286 INFO Thread-11 (_thread_body):2112019 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:40:25,286 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:25,286 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:40:25,287 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:25,287 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:40:25,287 INFO SenderThread:2112019 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:40:25,290 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:40:25,290 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:25,290 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:40:25,290 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:25,290 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:40:25,290 INFO SenderThread:2112019 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:40:25,361 INFO SenderThread:2112019 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:40:25,361 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:40:25,361 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:25,361 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:40:25,361 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:25,361 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:40:25,361 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:40:25,361 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:25,361 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:40:25,362 DEBUG SenderThread:2112019 [sender.py:send():379] send: final +2025-01-06 20:40:25,362 DEBUG SenderThread:2112019 [sender.py:send():379] send: footer +2025-01-06 20:40:25,362 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:25,362 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:40:25,362 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:25,362 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:25,363 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:40:25,363 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:25,363 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:40:25,365 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:40:25,365 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:40:25,365 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:25,412 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:25,412 INFO MainThread:2112019 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:40:25,413 INFO MainThread:2112019 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:40:25,413 INFO MainThread:2112019 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:40:25,413 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:40:25,413 INFO HandlerThread:2112019 [handler.py:finish():882] shutting down handler +2025-01-06 20:40:26,363 INFO WriterThread:2112019 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/run-vd7nl959.wandb +2025-01-06 20:40:26,412 INFO SenderThread:2112019 [sender.py:finish():1615] shutting down sender +2025-01-06 20:40:26,412 INFO SenderThread:2112019 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:40:26,412 INFO SenderThread:2112019 [file_pusher.py:join():175] waiting for file pusher diff --git a/vd7nl959/wandb/debug.log b/vd7nl959/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9ad8c8eb300e70e04d197e156d1c535c9ac06124 --- /dev/null +++ b/vd7nl959/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:35:57,438 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:35:57,438 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Configure stats pid to 2111854 +2025-01-06 20:35:57,438 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:35:57,438 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/settings +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug.log +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug-internal.log +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959'} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:init():619] starting backend +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:init():623] setting up manager +2025-01-06 20:35:57,445 INFO MainThread:2111854 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:35:57,446 INFO MainThread:2111854 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:35:57,458 INFO MainThread:2111854 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:35:57,464 INFO MainThread:2111854 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:35:57,719 INFO MainThread:2111854 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:35:57,756 INFO MainThread:2111854 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:35:57,757 INFO MainThread:2111854 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:35:58,159 INFO MainThread:2111854 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:35:58,159 INFO MainThread:2111854 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:35:58,159 INFO MainThread:2111854 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:35:58,160 INFO MainThread:2111854 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:35:58,162 INFO MainThread:2111854 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:35:58,162 INFO MainThread:2111854 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/tb, True +2025-01-06 20:36:02,209 INFO MainThread:2111854 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:40:26,416 WARNING MsgRouterThr:2111854 [router.py:message_loop():77] message_loop has been closed diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code/FISH/eval_robot.py b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ac48e8104761322e57b369c4bee21eac70facfd --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736213757 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/diff.patch b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c28171b0ccbf1f8e5d0d27b96acf339a03de1c06 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..a719175 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..bf34235b489a6e5ec51578c4d27b7c5aff91730c --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[10, 20, 0.5, 0.24900326137794287, 0.748354494522015, 0.297579273621989, 0.7263675480628274, 0.32785302404031924, 0.6719539165152919, 0.34575230631336995, 0.6572977080626605]]} \ No newline at end of file diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2527dd5ece45c104983ede3721a9441dac646550 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb31d93c1da1119a0f99dead6e8de7cc9426ba19ba355cdda44cdd7ec53264b +size 1142369 diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4dab3df107df2b9074d85fc54b5f69fdd04ac13e --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04da90b434a68619a5e33317b744b174f46337bf762360953ceab2e0bccc14ed +size 1142628 diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67e4792cbed72fe4bd73f72e0a30288e45c5b4e1 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf3e651ced546db728b0637e20b6faa1b21ce41d2776b443e9772929b3e1146c +size 1149554 diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aba4c4aa0a3d05bfbccd5dbf133e5d5606bb5947 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a4bd86fb7d6eb3cf5e41f6b667df345e5062e1c80bf8233f805f14559c1698d +size 1151827 diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..910e643a42675f6a17c286a79a48195e363f2930 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7007aef1a567b8508ad9494ca8afb542555722db42b465344bd8015a29ef78bd +size 1148908 diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8650fcb0d6ddbe4eecb5e0a1d3b9675b9a82580d --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log @@ -0,0 +1,268 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736213762.209845]: resetting environment +[INFO] [1736213762.213835]: cleared current plan +[INFO] [1736213762.214742]: moving to home +[INFO] [1736213763.216916]: reached home +[INFO] [1736213763.218299]: reset action history +[INFO] [1736213765.926447]: environment reset +Starting episode 0 +[INFO] [1736213765.928360]: resetting environment +[INFO] [1736213765.933200]: cleared current plan +[INFO] [1736213765.934184]: moving to home +[INFO] [1736213766.936514]: reached home +[INFO] [1736213766.937830]: reset action history +[INFO] [1736213769.646602]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736213784.913400]: resetting environment +[INFO] [1736213784.918558]: cleared current plan +[INFO] [1736213784.919903]: moving to home +[INFO] [1736213785.922322]: reached home +[INFO] [1736213785.923648]: reset action history +[INFO] [1736213788.629987]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/episode_rosbags/episode_0_2025-01-06-20-36-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2111854.log +started roslaunch server http://158.130.50.37:37355/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2112121] +process[print_text-2]: started with pid [2112145] +process[rosbag_record-3]: started with pid [2112146] +started bagging! +[INFO] [1736213815.640774]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736213816.376316]: Stored episode 1. +[INFO] [1736213816.377081]: Saving video... +[INFO] [1736213816.570918]: Video saved! +Starting episode 1 +[INFO] [1736213816.585218]: resetting environment +[INFO] [1736213816.589535]: cleared current plan +[INFO] [1736213816.589840]: moving to home +[INFO] [1736213820.293263]: reached home +[INFO] [1736213820.293891]: reset action history +[INFO] [1736213823.001755]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213824.917311]: resetting environment +[INFO] [1736213824.922070]: cleared current plan +[INFO] [1736213824.923016]: moving to home +[INFO] [1736213825.924459]: reached home +[INFO] [1736213825.925214]: reset action history +[INFO] [1736213828.632183]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/episode_rosbags/episode_1_2025-01-06-20-37-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2111854.log +started roslaunch server http://158.130.50.37:42713/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2112254] +process[print_text-5]: started with pid [2112255] +process[rosbag_record-6]: started with pid [2112279] +started bagging! +[INFO] [1736213855.596317]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736213856.353183]: Stored episode 2. +[INFO] [1736213856.353544]: Saving video... +[INFO] [1736213856.726380]: Video saved! +Starting episode 2 +[INFO] [1736213856.758264]: resetting environment +[INFO] [1736213856.765638]: cleared current plan +[INFO] [1736213856.767446]: moving to home +[INFO] [1736213860.571709]: reached home +[INFO] [1736213860.578462]: reset action history +[INFO] [1736213863.380989]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213877.427092]: resetting environment +[INFO] [1736213877.434995]: cleared current plan +[INFO] [1736213877.435197]: moving to home +[INFO] [1736213878.436871]: reached home +[INFO] [1736213878.437297]: reset action history +[INFO] [1736213881.148365]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/episode_rosbags/episode_2_2025-01-06-20-38-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2111854.log +started roslaunch server http://158.130.50.37:36871/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2112411] +process[print_text-8]: started with pid [2112412] +process[rosbag_record-9]: started with pid [2112413] +started bagging! +For topic gripper_width: timestamp difference is 119264915 for nearest: 1736213883933221176 - target: 1736213883813956261 at idx 43 +For topic gripper_width: timestamp difference is 101739358 for nearest: 1736213903333232593 - target: 1736213903231493235 at idx 249 +[INFO] [1736213908.316826]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736213909.071840]: Stored episode 3. +[INFO] [1736213909.074603]: Saving video... +[INFO] [1736213909.637250]: Video saved! +Starting episode 3 +[INFO] [1736213909.670675]: resetting environment +[INFO] [1736213909.685220]: cleared current plan +[INFO] [1736213909.686852]: moving to home +[WARN] [1736213909.648687]: Plan exhausted +[INFO] [1736213913.498701]: reached home +[INFO] [1736213913.501701]: reset action history +[INFO] [1736213916.308744]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213921.160169]: resetting environment +[INFO] [1736213921.175417]: cleared current plan +[INFO] [1736213921.177351]: moving to home +[INFO] [1736213922.180524]: reached home +[INFO] [1736213922.182376]: reset action history +[INFO] [1736213925.081959]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/episode_rosbags/episode_3_2025-01-06-20-38-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2111854.log +started roslaunch server http://158.130.50.37:41253/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2112549] +process[print_text-11]: started with pid [2112550] +process[rosbag_record-12]: started with pid [2112551] +started bagging! +For topic gripper_width: timestamp difference is 103279899 for nearest: 1736213944899926971 - target: 1736213944796647072 at idx 249 +For topic gripper_width: timestamp difference is 103673331 for nearest: 1736213945099936597 - target: 1736213944996263266 at idx 249 +For topic EE_pose: timestamp difference is 180268676 for nearest: 1736213947310142190 - target: 1736213947129873514 at idx 0 +[INFO] [1736213952.280829]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736213953.060112]: Stored episode 4. +[INFO] [1736213953.069372]: Saving video... +[INFO] [1736213953.584371]: Video saved! +Starting episode 4 +[INFO] [1736213953.623927]: resetting environment +[INFO] [1736213953.647798]: cleared current plan +[INFO] [1736213953.651734]: moving to home +[INFO] [1736213957.273115]: reached home +[INFO] [1736213957.278239]: reset action history +[INFO] [1736213960.087224]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213961.808540]: resetting environment +[INFO] [1736213961.822696]: cleared current plan +[INFO] [1736213961.826431]: moving to home +[INFO] [1736213962.832228]: reached home +[INFO] [1736213962.837509]: reset action history +[INFO] [1736213965.613107]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/episode_rosbags/episode_4_2025-01-06-20-39-25.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2111854.log +started roslaunch server http://158.130.50.37:36391/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2112660] +process[print_text-14]: started with pid [2112661] +process[rosbag_record-15]: started with pid [2112685] +started bagging! +For topic gripper_width: timestamp difference is 103120267 for nearest: 1736213981599899232 - target: 1736213981496778965 at idx 204 +For topic gripper_width: timestamp difference is -107994413 for nearest: 1736213985833261872 - target: 1736213985941256285 at idx 249 +For topic gripper_width: timestamp difference is 100916151 for nearest: 1736213990399926666 - target: 1736213990299010515 at idx 249 +[INFO] [1736213992.585150]: Storing episode... +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[INFO] [1736213993.375608]: Stored episode 5. +[INFO] [1736213993.384497]: Saving video... +[INFO] [1736213993.840622]: Video saved! +[INFO] [1736213993.885660]: resetting environment +[INFO] [1736213993.900446]: cleared current plan +[INFO] [1736213993.901371]: moving to home +[INFO] [1736213997.712800]: reached home +[INFO] [1736213997.725163]: reset action history +[INFO] [1736214000.585990]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/requirements.txt b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-metadata.json b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..615fca223e90156a71840eaf24b8534094bb3fb7 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T01:35:57.823325", + "startedAt": "2025-01-07T01:35:57.437487", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.21275, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.98, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.61, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 505.5578384399414 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..395260a3ab89bd3c8e988e7a63437244d0869531 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "1eb31d93c1da1119a0f99dead6e8de7cc9426ba19ba355cdda44cdd7ec53264b", "size": 1142369, "path": "media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4"}, "global_step": 665, "_timestamp": 1736214003.2681391, "_runtime": 245.82163906097412, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "04da90b434a68619a5e33317b744b174f46337bf762360953ceab2e0bccc14ed", "size": 1142628, "path": "media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "bf3e651ced546db728b0637e20b6faa1b21ce41d2776b443e9772929b3e1146c", "size": 1149554, "path": "media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "9a4bd86fb7d6eb3cf5e41f6b667df345e5062e1c80bf8233f805f14559c1698d", "size": 1151827, "path": "media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "7007aef1a567b8508ad9494ca8afb542555722db42b465344bd8015a29ef78bd", "size": 1148908, "path": "media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://rd5slrhneu6v1cf9h2xlvgv5h1omqi7mzjkyq221ljbvdpid4cbun2gn8diz7gtve9rgrqh3ol19ozmhhc2bi2eecp4ky8sh0mw0swrl4ky9zbeonjb6tx2rbwpd5gi7/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://es1uulaq0k2tj5bcr8ileuqfkcuuq97j1gkpr1z238owbjyfeij9vfggveuhwhkrxcmv4sqsth2axno5fcbrdjoo4cmfk03u334o7la9xgewiy1ejzg8pj0f94t2t6rj:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 20, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.24900326137794287, "total_success_rate_ub_0.01": 0.748354494522015, "total_success_rate_lb_0.025": 0.297579273621989, "total_success_rate_ub_0.025": 0.7263675480628274, "total_success_rate_lb_0.05": 0.32785302404031924, "total_success_rate_ub_0.05": 0.6719539165152919, "total_success_rate_lb_0.1": 0.34575230631336995, "total_success_rate_ub_0.1": 0.6572977080626605, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "633be57ab8f5d2a03a6ed414b12b88c46f9388363d35325712a36769135ab23a", "size": 506, "artifact_path": "wandb-client-artifact://wig5ec2k7h1ygwe2rng55729453lg1tx23aafomuetiayt5nfi1qizz7gj6hq3dx66qhj1y3s4r7u6cs2zuxjniu326f038f792k3fugn7gnkfwadim5405nffl79ovb/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vzuw0akgikdcuuagbp6wto7zg6jpyy80m0gr45bqzme7hx7aimsybesw0vzpb3oui2niw9ldyizwfnp328nx276xqf4elmizl8igc7shh72jz7zeeff2eg61wxyvguf9:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 255}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.24900326132774353, "eval/total_success_rate_ub_0.01": 0.7483544945716858, "eval/total_success_rate_lb_0.025": 0.297579288482666, "eval/total_success_rate_ub_0.025": 0.7263675332069397, "eval/total_success_rate_lb_0.05": 0.3278530240058899, "eval/total_success_rate_ub_0.05": 0.6719539165496826, "eval/total_success_rate_lb_0.1": 0.3457522988319397, "eval/total_success_rate_ub_0.1": 0.6572977304458618} \ No newline at end of file diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug-internal.log b/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0c77048b4f24d32be1c4af49c0a8b5a3388b0315 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug-internal.log @@ -0,0 +1,753 @@ +2025-01-06 20:35:57,446 INFO StreamThr :2112019 [internal.py:wandb_internal():85] W&B internal server running at pid: 2112019, started at: 2025-01-06 20:35:57.445835 +2025-01-06 20:35:57,447 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:35:57,458 INFO WriterThread:2112019 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/run-vd7nl959.wandb +2025-01-06 20:35:57,459 DEBUG SenderThread:2112019 [sender.py:send():379] send: header +2025-01-06 20:35:57,466 DEBUG SenderThread:2112019 [sender.py:send():379] send: run +2025-01-06 20:35:57,702 INFO SenderThread:2112019 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files +2025-01-06 20:35:57,702 INFO SenderThread:2112019 [sender.py:_start_run_threads():1188] run started: vd7nl959 with start time 1736213757.4465 +2025-01-06 20:35:57,720 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:35:57,720 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:35:57,805 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:35:57,810 DEBUG HandlerThread:2112019 [system_info.py:__init__():26] System info init +2025-01-06 20:35:57,810 DEBUG HandlerThread:2112019 [system_info.py:__init__():41] System info init done +2025-01-06 20:35:57,810 INFO HandlerThread:2112019 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:35:57,810 INFO SystemMonitor:2112019 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:35:57,810 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:35:57,811 INFO HandlerThread:2112019 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:35:57,812 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:35:57,812 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:35:57,813 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:35:57,813 INFO SystemMonitor:2112019 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:35:57,823 DEBUG HandlerThread:2112019 [system_info.py:probe():152] Probing system +2025-01-06 20:35:57,824 DEBUG HandlerThread:2112019 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:35:57,829 DEBUG HandlerThread:2112019 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:35:57,829 DEBUG HandlerThread:2112019 [system_info.py:probe():200] Probing system done +2025-01-06 20:35:57,829 DEBUG HandlerThread:2112019 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:35:57.823325', 'startedAt': '2025-01-07T01:35:57.437487', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.21275, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.98, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.61, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 505.5578384399414}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:35:57,829 INFO HandlerThread:2112019 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:35:57,829 INFO HandlerThread:2112019 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:35:57,829 DEBUG HandlerThread:2112019 [system_info.py:_save_code():44] Saving code +2025-01-06 20:35:57,831 DEBUG HandlerThread:2112019 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:35:57,831 DEBUG HandlerThread:2112019 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:35:57,915 DEBUG HandlerThread:2112019 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:35:57,915 INFO HandlerThread:2112019 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:35:57,918 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:35:57,918 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:35:57,919 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:35:57,919 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:35:58,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:35:58,159 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:35:58,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:58,161 DEBUG SenderThread:2112019 [sender.py:send():379] send: telemetry +2025-01-06 20:35:58,181 INFO wandb-upload_2:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/yfojxhab-diff.patch +2025-01-06 20:35:58,181 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/deoj5bdl-wandb-metadata.json +2025-01-06 20:35:58,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:35:58,201 INFO HandlerThread:2112019 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/tb" + save: true +} +control { + relay_id: "deba67f5c4a4425e96a5a9a7e46a27bb" +} +_info { + stream_id: "vd7nl959" +} + +2025-01-06 20:35:58,201 DEBUG HandlerThread:2112019 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:35:58,202 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:35:58,218 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/433434gp-code/FISH/eval_robot.py +2025-01-06 20:35:58,275 DEBUG SenderThread:2112019 [sender.py:send():379] send: tbrecord +2025-01-06 20:35:58,276 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:35:58,276 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 with policy live +2025-01-06 20:35:58,513 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/e2xw3ogu-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:35:58,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:35:58,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-metadata.json +2025-01-06 20:35:58,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code/FISH/eval_robot.py +2025-01-06 20:35:58,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/diff.patch +2025-01-06 20:35:58,705 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/requirements.txt +2025-01-06 20:35:58,705 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:35:58,705 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code +2025-01-06 20:35:58,705 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code/FISH +2025-01-06 20:35:59,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:00,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:00,703 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:01,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:02,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:02,209 DEBUG SenderThread:2112019 [sender.py:send():379] send: config +2025-01-06 20:36:03,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:03,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:04,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:04,704 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:05,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:06,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:07,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:08,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:08,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:08,706 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:09,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:10,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:10,706 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:11,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:12,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:13,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:36:13,159 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:36:13,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:13,295 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:14,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:15,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:16,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:17,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:18,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:19,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:19,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:20,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:21,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:22,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:23,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:24,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:24,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:25,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:26,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:26,710 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:27,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:28,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:36:28,159 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:36:28,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:28,711 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:29,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:29,934 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:30,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:30,712 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml +2025-01-06 20:36:30,712 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:31,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:32,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:33,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:34,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:35,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:35,164 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:36,164 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:37,164 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:38,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:39,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:40,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:41,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:41,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:42,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:43,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:36:43,159 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:36:43,205 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:44,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:45,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:46,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:46,310 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:47,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:48,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:49,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:50,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:51,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:52,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:52,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:53,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:54,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:55,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:56,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:56,583 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:36:56,583 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 with policy now +2025-01-06 20:36:56,583 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:36:56,584 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:36:56,584 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:36:56,584 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos +2025-01-06 20:36:56,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:36:56,848 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/pwh2lllx-media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 +2025-01-06 20:36:57,114 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/be2qygjy-media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 +2025-01-06 20:36:57,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:57,590 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:36:57,813 DEBUG SystemMonitor:2112019 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:36:57,814 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:36:58,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:36:58,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:36:58,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:36:58,719 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:36:59,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:00,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:01,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:02,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:02,721 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:03,002 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:03,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:04,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:04,722 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:05,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:05,722 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:37:05,974 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/1czzhzeo-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:37:06,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:06,722 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:07,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:08,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:08,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:08,723 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:09,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:10,169 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:10,724 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:11,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:12,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:13,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:37:13,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:37:13,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:13,278 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:14,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:15,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:16,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:17,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:18,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:19,162 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:19,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:20,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:21,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:22,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:23,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:24,163 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:24,171 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:25,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:26,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:27,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:27,814 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:37:28,159 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:37:28,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:37:28,201 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:29,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:29,324 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:30,173 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:31,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:32,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:33,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:34,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:35,166 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:35,175 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:36,175 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:36,732 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:36,755 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:37:36,755 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 with policy now +2025-01-06 20:37:36,756 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:37:36,768 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:37:36,769 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:37:36,769 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:37:37,005 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/jiyvyyvk-media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 +2025-01-06 20:37:37,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:37,732 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:37:37,732 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 +2025-01-06 20:37:37,733 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:37:38,024 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/1749qcx4-media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 +2025-01-06 20:37:38,178 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:38,733 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:39,180 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:40,178 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:40,180 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:41,181 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:42,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:42,734 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:43,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:37:43,166 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:37:43,209 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:44,181 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:44,735 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:45,181 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:45,421 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:46,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:47,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:48,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:49,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:50,182 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:51,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:51,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:52,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:53,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:54,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:55,183 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:56,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:57,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:37:57,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:57,739 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:37:57,815 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:37:58,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:37:58,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:37:58,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:37:58,660 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:37:58,660 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:37:58,661 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:37:58,739 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:37:58,740 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:37:59,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:00,184 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:00,741 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:01,189 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:02,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:02,579 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:02,741 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:03,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:04,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:04,742 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:05,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:06,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:07,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:07,978 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:08,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:09,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:10,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:11,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:12,185 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:13,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:38:13,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:38:13,205 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:13,251 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:14,186 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:15,186 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:16,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:17,186 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:18,186 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:19,167 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:19,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:20,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:21,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:22,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:23,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:24,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:24,381 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:24,749 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:25,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:26,187 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:27,188 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:27,815 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:38:28,160 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:38:28,160 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:38:28,205 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:29,190 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:29,639 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:29,681 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:38:29,682 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:38:29,682 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 with policy now +2025-01-06 20:38:29,682 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:38:29,682 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:38:29,683 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:38:29,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:38:29,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 +2025-01-06 20:38:29,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:38:29,944 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/3lx7aqdl-media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 +2025-01-06 20:38:30,194 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:30,214 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/00un4lro-media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 +2025-01-06 20:38:30,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:31,197 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:31,751 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:32,202 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:33,210 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:34,204 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:34,752 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:35,203 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:35,541 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:36,204 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:37,207 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:38,207 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:38,754 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:39,208 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:40,208 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:41,136 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:41,206 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:41,452 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:38:41,452 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:38:41,453 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:38:41,755 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:38:41,755 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:38:42,007 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/yyhpk17r-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:38:42,229 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:42,755 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:43,165 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:38:43,165 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:38:43,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:44,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:44,756 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:45,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:46,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:46,574 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:46,757 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:38:47,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:48,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:49,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:50,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:51,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:52,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:52,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:53,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:54,213 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:55,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:56,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:57,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:57,816 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:38:57,816 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:38:58,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:38:58,161 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:38:58,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:38:59,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:00,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:01,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:02,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:03,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:03,214 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:04,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:05,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:06,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:06,763 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:07,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:08,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:08,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:08,764 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:09,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:10,215 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:11,217 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:12,216 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:13,164 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:39:13,164 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:39:13,221 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:13,324 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:13,629 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:39:13,629 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:39:13,630 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 with policy now +2025-01-06 20:39:13,630 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:39:13,630 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:39:13,630 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:39:13,766 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:39:13,766 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 +2025-01-06 20:39:13,766 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:39:13,908 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/1vzzwz6d-media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 +2025-01-06 20:39:14,165 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/8pvkcqip-media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 +2025-01-06 20:39:14,243 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:14,766 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:15,224 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:16,224 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:17,237 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:18,237 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:18,768 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:19,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:19,240 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:20,225 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:21,230 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:22,231 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:22,506 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:39:22,506 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:39:22,506 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:39:22,769 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:39:22,769 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:22,769 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:39:23,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:24,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:24,770 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:25,172 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:25,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:26,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:26,770 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:27,231 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:27,816 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:39:28,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:39:28,161 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:39:28,231 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:29,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:30,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:30,276 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:31,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:32,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:33,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:34,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:35,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:36,170 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:36,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:37,232 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:38,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:39,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:40,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:41,174 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:41,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:42,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:42,776 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:43,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:39:43,161 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:39:43,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:44,233 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:45,234 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:46,234 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:46,777 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:47,073 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:47,234 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:48,234 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:49,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:50,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:51,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:52,235 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:52,449 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:52,780 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:53,243 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:53,883 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:39:53,883 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 with policy now +2025-01-06 20:39:53,885 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:39:53,886 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:39:53,886 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:39:53,887 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:39:54,161 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/1dcty9no-media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 +2025-01-06 20:39:54,236 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:54,781 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:54,781 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:39:54,781 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 +2025-01-06 20:39:54,781 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval +2025-01-06 20:39:55,088 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/ew2wcu23-media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 +2025-01-06 20:39:55,248 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:56,245 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:57,244 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:57,718 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:39:57,817 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:39:58,168 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:39:58,168 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:39:58,254 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:39:58,782 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:39:59,266 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:00,257 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:01,254 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:02,254 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:02,555 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:40:02,555 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:02,555 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:02,783 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:02,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:02,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:02,784 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:40:02,784 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:40:02,978 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:40:03,029 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/ilxnxfmq-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:03,260 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:03,275 INFO wandb-upload_0:2112019 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn4nq66mv +2025-01-06 20:40:03,366 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:40:03,375 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:40:03,675 INFO SenderThread:2112019 [sender.py:send_request_log_artifact():1525] logged artifact run-vd7nl959-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMwMjk3Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYzNzE0OQ==', 'latestArtifact': None}} +2025-01-06 20:40:03,675 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:40:03,675 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:40:03,676 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:40:03,676 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,676 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:03,676 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,677 DEBUG SenderThread:2112019 [sender.py:send():379] send: telemetry +2025-01-06 20:40:03,677 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,680 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,680 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,681 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,681 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,681 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,682 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,682 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,682 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,682 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,683 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,683 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,683 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,683 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,684 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,684 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,684 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,684 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,684 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,685 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,685 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:03,685 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:03,685 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:40:03,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media +2025-01-06 20:40:03,785 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table +2025-01-06 20:40:03,785 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval +2025-01-06 20:40:03,867 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/2ne47f8i-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:40:04,008 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/6bd135am-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2025-01-06 20:40:04,084 INFO wandb-upload_2:2112019 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4imv0blj +2025-01-06 20:40:04,110 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/5f4vy0vc-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:40:04,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:04,515 INFO SenderThread:2112019 [sender.py:send_request_log_artifact():1525] logged artifact run-vd7nl959-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjMwMjk5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYzNzE1MA==', 'latestArtifact': None}} +2025-01-06 20:40:04,515 DEBUG SenderThread:2112019 [sender.py:send():379] send: files +2025-01-06 20:40:04,515 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json with policy now +2025-01-06 20:40:04,516 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:40:04,516 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:04,516 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:04,781 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/rndff0hh-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2025-01-06 20:40:04,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:04,784 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:05,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:06,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:07,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:08,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:09,176 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:09,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:10,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:11,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:12,255 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:13,161 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:40:13,162 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:40:13,256 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:13,609 DEBUG SenderThread:2112019 [sender.py:send():379] send: exit +2025-01-06 20:40:13,610 INFO SenderThread:2112019 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:40:13,610 INFO SenderThread:2112019 [sender.py:send_exit():588] handling runtime: 255 +2025-01-06 20:40:13,611 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:13,611 INFO SenderThread:2112019 [sender.py:send_exit():594] send defer +2025-01-06 20:40:13,611 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,611 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:40:13,611 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:13,611 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:40:13,611 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:40:13,611 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,611 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:40:13,612 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:13,612 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:40:13,612 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:40:13,612 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,612 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:40:13,612 INFO HandlerThread:2112019 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:40:13,612 DEBUG SystemMonitor:2112019 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:40:13,612 DEBUG SystemMonitor:2112019 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:40:13,613 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:40:13,613 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:40:13,664 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:40:13,664 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:40:13,664 INFO HandlerThread:2112019 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:40:13,665 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:13,665 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:40:13,665 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:40:13,665 DEBUG SenderThread:2112019 [sender.py:send():379] send: stats +2025-01-06 20:40:13,665 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,665 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:40:13,665 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:13,665 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:40:13,665 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:40:13,666 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:13,666 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:40:13,788 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:14,789 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:20,562 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:20,562 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:20,562 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:40:20,562 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:20,563 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:20,563 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:40:20,563 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:40:20,563 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:20,563 DEBUG SenderThread:2112019 [sender.py:send():379] send: history +2025-01-06 20:40:20,564 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:40:20,564 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:20,564 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:40:20,565 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:20,565 DEBUG SenderThread:2112019 [sender.py:send():379] send: summary +2025-01-06 20:40:20,565 INFO SenderThread:2112019 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:40:20,565 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:20,565 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:40:20,565 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:40:20,565 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:20,566 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:40:20,566 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:20,566 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:40:20,566 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:40:20,566 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:40:20,566 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:20,566 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:40:20,566 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:20,566 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:40:20,611 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:20,791 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml +2025-01-06 20:40:20,791 INFO Thread-12 :2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:23,793 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:40:23,793 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:23,793 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:23,794 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:40:23,794 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:23,794 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:40:23,794 INFO SenderThread:2112019 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:40:23,796 INFO SenderThread:2112019 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:40:23,798 INFO SenderThread:2112019 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:40:23,813 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:40:23,813 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:23,813 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:40:23,813 DEBUG SenderThread:2112019 [sender.py:send():379] send: artifact +2025-01-06 20:40:24,101 INFO SenderThread:2112019 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:40:24,101 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:24,101 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:40:24,101 INFO SenderThread:2112019 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:40:24,612 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:24,792 INFO SenderThread:2112019 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:24,792 INFO SenderThread:2112019 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml config.yaml +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json wandb-summary.json +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/requirements.txt requirements.txt +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log output.log +2025-01-06 20:40:24,793 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/diff.patch diff.patch +2025-01-06 20:40:24,795 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:40:24,798 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 media/videos/eval/1_eval_1_04da90b434a68619a5e3.mp4 +2025-01-06 20:40:24,798 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 media/videos/eval/2_eval_2_bf3e651ced546db728b0.mp4 +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 media/videos/eval/3_eval_3_9a4bd86fb7d6eb3cf5e4.mp4 +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 media/videos/eval/0_eval_0_1eb31d93c1da1119a0f9.mp4 +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 media/videos/eval/4_eval_4_7007aef1a567b8508ad9.mp4 +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:40:24,799 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:40:24,799 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:24,799 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:24,799 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:40:24,801 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:24,801 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:40:24,801 INFO SenderThread:2112019 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:40:24,988 INFO wandb-upload_2:2112019 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/requirements.txt +2025-01-06 20:40:24,999 INFO wandb-upload_0:2112019 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/wandb-summary.json +2025-01-06 20:40:25,010 INFO wandb-upload_1:2112019 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/config.yaml +2025-01-06 20:40:25,041 INFO wandb-upload_4:2112019 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/files/output.log +2025-01-06 20:40:25,086 INFO wandb-upload_3:2112019 [upload_job.py:push():130] Uploaded file /tmp/tmpvd_mqijuwandb/holtclnn-events.out.tfevents.1736213758.leonmkim-ROG-Strix-G15CS-G15CS.2111854.0 +2025-01-06 20:40:25,286 INFO Thread-11 (_thread_body):2112019 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:40:25,286 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:25,286 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:40:25,287 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:25,287 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:40:25,287 INFO SenderThread:2112019 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:40:25,290 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:40:25,290 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:25,290 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:40:25,290 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:25,290 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:40:25,290 INFO SenderThread:2112019 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:40:25,361 INFO SenderThread:2112019 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:40:25,361 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:40:25,361 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:25,361 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:40:25,361 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:25,361 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:40:25,361 INFO SenderThread:2112019 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:40:25,361 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:40:25,361 INFO HandlerThread:2112019 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:40:25,362 DEBUG SenderThread:2112019 [sender.py:send():379] send: final +2025-01-06 20:40:25,362 DEBUG SenderThread:2112019 [sender.py:send():379] send: footer +2025-01-06 20:40:25,362 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: defer +2025-01-06 20:40:25,362 INFO SenderThread:2112019 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:40:25,362 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:25,362 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:25,363 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:40:25,363 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:40:25,363 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:40:25,365 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:40:25,365 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:40:25,365 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:40:25,412 DEBUG SenderThread:2112019 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:40:25,412 INFO MainThread:2112019 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:40:25,413 INFO MainThread:2112019 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:40:25,413 INFO MainThread:2112019 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:40:25,413 DEBUG HandlerThread:2112019 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:40:25,413 INFO HandlerThread:2112019 [handler.py:finish():882] shutting down handler +2025-01-06 20:40:26,363 INFO WriterThread:2112019 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/run-vd7nl959.wandb +2025-01-06 20:40:26,412 INFO SenderThread:2112019 [sender.py:finish():1615] shutting down sender +2025-01-06 20:40:26,412 INFO SenderThread:2112019 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:40:26,412 INFO SenderThread:2112019 [file_pusher.py:join():175] waiting for file pusher diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug.log b/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9ad8c8eb300e70e04d197e156d1c535c9ac06124 --- /dev/null +++ b/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:35:57,438 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:35:57,438 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Configure stats pid to 2111854 +2025-01-06 20:35:57,438 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:35:57,438 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/settings +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug.log +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/wandb/run-20250106_203557-vd7nl959/logs/debug-internal.log +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959'} +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:init():619] starting backend +2025-01-06 20:35:57,439 INFO MainThread:2111854 [wandb_init.py:init():623] setting up manager +2025-01-06 20:35:57,445 INFO MainThread:2111854 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:35:57,446 INFO MainThread:2111854 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:35:57,458 INFO MainThread:2111854 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:35:57,464 INFO MainThread:2111854 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:35:57,719 INFO MainThread:2111854 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:35:57,756 INFO MainThread:2111854 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:35:57,757 INFO MainThread:2111854 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:35:58,159 INFO MainThread:2111854 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:35:58,159 INFO MainThread:2111854 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:35:58,159 INFO MainThread:2111854 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:35:58,160 INFO MainThread:2111854 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:35:58,162 INFO MainThread:2111854 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:35:58,162 INFO MainThread:2111854 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/vd7nl959/tb, True +2025-01-06 20:36:02,209 INFO MainThread:2111854 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:40:26,416 WARNING MsgRouterThr:2111854 [router.py:message_loop():77] message_loop has been closed diff --git a/vd7nl959/wandb/run-20250106_203557-vd7nl959/run-vd7nl959.wandb b/vd7nl959/wandb/run-20250106_203557-vd7nl959/run-vd7nl959.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7f63156f1327fe33020944f334bd6861973c98a2 Binary files /dev/null and b/vd7nl959/wandb/run-20250106_203557-vd7nl959/run-vd7nl959.wandb differ diff --git a/xgtu747n/.hydra/config.yaml b/xgtu747n/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/xgtu747n/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/xgtu747n/.hydra/hydra.yaml b/xgtu747n/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7ac258ce1fbe62fb66d8270a039316e395d1f1a --- /dev/null +++ b/xgtu747n/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/xgtu747n + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/xgtu747n/.hydra/overrides.yaml b/xgtu747n/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/xgtu747n/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/xgtu747n/eval_policy.log b/xgtu747n/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..684dd39e9a66ecc47dcd5a8e2a9c707e3117c6c3 --- /dev/null +++ b/xgtu747n/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 22:18:04,636][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:18:04,639][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 22:18:05,794][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/z85umdwq/.hydra/config.yaml b/z85umdwq/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/z85umdwq/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/z85umdwq/.hydra/hydra.yaml b/z85umdwq/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a50f0824c97a78ad1a173635d7aeefded7860cb --- /dev/null +++ b/z85umdwq/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/z85umdwq + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/z85umdwq/.hydra/overrides.yaml b/z85umdwq/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/z85umdwq/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/z85umdwq/eval_policy.log b/z85umdwq/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..7809bc87bf45787396eab3a869a8b35766cc390e --- /dev/null +++ b/z85umdwq/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 16:58:23,481][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 16:58:23,485][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 16:58:24,612][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) +