diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..7d7a86c5dc17013ce3ada8df73f03591308d9a25 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,39 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3fs5lhth/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +urh96fgp/episode_rosbags/episode_2_2025-01-07-20-13-17.bag filter=lfs diff=lfs merge=lfs -text +kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 filter=lfs diff=lfs merge=lfs -text +kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 filter=lfs diff=lfs merge=lfs -text +kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 filter=lfs diff=lfs merge=lfs -text +kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 filter=lfs diff=lfs merge=lfs -text +kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/3fs5lhth/.hydra/config.yaml b/3fs5lhth/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/3fs5lhth/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/3fs5lhth/.hydra/hydra.yaml b/3fs5lhth/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af39e3689897e1587f3b21559b7ac5c1270f0823 --- /dev/null +++ b/3fs5lhth/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/3fs5lhth/.hydra/overrides.yaml b/3fs5lhth/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/3fs5lhth/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/3fs5lhth/episode_rosbags/aligned_depth_to_color_K.npy b/3fs5lhth/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3fs5lhth/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3fs5lhth/episode_rosbags/cam_tf_world.npy b/3fs5lhth/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/3fs5lhth/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/3fs5lhth/episode_rosbags/color_K.npy b/3fs5lhth/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3fs5lhth/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3fs5lhth/episode_rosbags/depth_K.npy b/3fs5lhth/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/3fs5lhth/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/3fs5lhth/eval_robot.log b/3fs5lhth/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..bf34efbf97d81a537b2bfb747fc7420bb824c79b --- /dev/null +++ b/3fs5lhth/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 22:57:11,123][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:57:11,126][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/3fs5lhth/eval_video/0_eval.mp4 b/3fs5lhth/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea2b57b7e1ebcdd4bccedeb41ea80b0ab9f89d89 --- /dev/null +++ b/3fs5lhth/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b48560eafc6e2046c0d3c86150f45f655df98b25eb482e7512147ab646dd46e +size 1518923 diff --git a/3fs5lhth/eval_video/1_eval.mp4 b/3fs5lhth/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0688555e43b42137c1749d8d0eee120af55c2547 --- /dev/null +++ b/3fs5lhth/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:487641939ef48b883b67f94fbb0868673b10eff8a02668d21495242e2d202682 +size 1522323 diff --git a/3fs5lhth/eval_video/2_eval.mp4 b/3fs5lhth/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..548084be99b60754939ddbf1baeb777bc278aa3d --- /dev/null +++ b/3fs5lhth/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e00f3422b92e36dcbe3325b249715f22f7c8759ca190ee943a66de4faad718b8 +size 1497002 diff --git a/3fs5lhth/eval_video/3_eval.mp4 b/3fs5lhth/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f20423d600c4b66874fccc8a60c778394a701604 --- /dev/null +++ b/3fs5lhth/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebb2a03c3b35f1d334353e9bc4e14c41ed6a57812dc8d683059e0f5ba1d2244c +size 1471298 diff --git a/3fs5lhth/eval_video/4_eval.mp4 b/3fs5lhth/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d44df9795a46b58885ce5c5375a527929d8f01b1 --- /dev/null +++ b/3fs5lhth/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e244931479443b8b04bff9562fb1d38e7dd251741cb1dbeaa5f6e6ae33c06fc7 +size 1544494 diff --git a/3fs5lhth/tb/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 b/3fs5lhth/tb/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 new file mode 100644 index 0000000000000000000000000000000000000000..d29215b7e0cd6a96ffa37988730601b735fe6fa7 --- /dev/null +++ b/3fs5lhth/tb/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5d830069feaf878cc09a7ec4bdcae153bfac838fc54b1d7f380c0a75413fab5 +size 6539 diff --git a/3fs5lhth/wandb/debug-internal.log b/3fs5lhth/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..595c8a55fc5993644a0ca22639ca25f9c06ababa --- /dev/null +++ b/3fs5lhth/wandb/debug-internal.log @@ -0,0 +1,813 @@ +2025-01-06 22:57:16,956 INFO StreamThr :2132474 [internal.py:wandb_internal():85] W&B internal server running at pid: 2132474, started at: 2025-01-06 22:57:16.956330 +2025-01-06 22:57:16,957 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:57:16,969 INFO WriterThread:2132474 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/run-3fs5lhth.wandb +2025-01-06 22:57:16,969 DEBUG SenderThread:2132474 [sender.py:send():379] send: header +2025-01-06 22:57:16,976 DEBUG SenderThread:2132474 [sender.py:send():379] send: run +2025-01-06 22:57:17,225 INFO SenderThread:2132474 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files +2025-01-06 22:57:17,225 INFO SenderThread:2132474 [sender.py:_start_run_threads():1188] run started: 3fs5lhth with start time 1736222236.956636 +2025-01-06 22:57:17,241 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:57:17,242 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:57:17,288 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:57:17,293 DEBUG HandlerThread:2132474 [system_info.py:__init__():26] System info init +2025-01-06 22:57:17,293 DEBUG HandlerThread:2132474 [system_info.py:__init__():41] System info init done +2025-01-06 22:57:17,293 INFO HandlerThread:2132474 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:57:17,293 INFO SystemMonitor:2132474 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:57:17,293 INFO HandlerThread:2132474 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:57:17,293 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:57:17,294 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:57:17,295 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:57:17,295 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:57:17,296 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:57:17,306 DEBUG HandlerThread:2132474 [system_info.py:probe():152] Probing system +2025-01-06 22:57:17,307 DEBUG HandlerThread:2132474 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:57:17,312 DEBUG HandlerThread:2132474 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:57:17,312 DEBUG HandlerThread:2132474 [system_info.py:probe():200] Probing system done +2025-01-06 22:57:17,312 DEBUG HandlerThread:2132474 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:57:17.306273', 'startedAt': '2025-01-07T03:57:16.946794', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.0555, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.908, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 638.5600509643555}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:57:17,312 INFO HandlerThread:2132474 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:57:17,312 INFO HandlerThread:2132474 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:57:17,312 DEBUG HandlerThread:2132474 [system_info.py:_save_code():44] Saving code +2025-01-06 22:57:17,313 DEBUG HandlerThread:2132474 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:57:17,313 DEBUG HandlerThread:2132474 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:57:17,377 DEBUG HandlerThread:2132474 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:57:17,377 INFO HandlerThread:2132474 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:57:17,380 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 22:57:17,380 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:57:17,381 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:57:17,381 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:57:17,627 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/39oobfg6-wandb-metadata.json +2025-01-06 22:57:17,629 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:57:17,629 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:57:17,629 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:57:17,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:57:17,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:17,631 INFO HandlerThread:2132474 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/tb" + save: true +} +control { + relay_id: "bc2d8352641d45d99d74edbb889444b7" +} +_info { + stream_id: "3fs5lhth" +} + +2025-01-06 22:57:17,632 DEBUG HandlerThread:2132474 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:57:17,649 INFO wandb-upload_2:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/kv49xzv5-diff.patch +2025-01-06 22:57:17,652 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/639feo2f-code/FISH/eval_robot.py +2025-01-06 22:57:17,762 DEBUG SenderThread:2132474 [sender.py:send():379] send: telemetry +2025-01-06 22:57:17,763 DEBUG SenderThread:2132474 [sender.py:send():379] send: tbrecord +2025-01-06 22:57:17,763 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 22:57:17,763 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 with policy live +2025-01-06 22:57:17,951 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/hr5aukav-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:57:18,226 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/requirements.txt +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code/FISH/eval_robot.py +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-metadata.json +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/diff.patch +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code/FISH +2025-01-06 22:57:18,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:19,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:20,226 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:20,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:21,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:22,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:22,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:23,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:24,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:25,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:26,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:27,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:27,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:28,052 DEBUG SenderThread:2132474 [sender.py:send():379] send: config +2025-01-06 22:57:28,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:29,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:30,228 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:30,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:31,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:32,629 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:57:32,629 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:57:32,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:32,685 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:33,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:34,229 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:34,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:35,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:36,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:37,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:38,231 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:38,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:38,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:39,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:40,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:41,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:42,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:43,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:43,877 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:44,232 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:44,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:45,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:46,233 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:46,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:47,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:57:47,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:57:47,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:48,234 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:48,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:49,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:49,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:50,234 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml +2025-01-06 22:57:50,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:51,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:52,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:53,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:54,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:55,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:55,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:56,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:57,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:58,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:59,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:00,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:00,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:01,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:02,629 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:58:02,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:58:02,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:03,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:04,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:05,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:05,747 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:06,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:07,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:08,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:09,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:10,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:11,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:11,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:12,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:13,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:14,241 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:14,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:15,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:16,241 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:16,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:17,296 DEBUG SystemMonitor:2132474 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:58:17,297 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 22:58:17,298 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:17,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:58:17,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:58:17,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:18,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:19,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:20,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:21,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:22,440 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:22,450 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 22:58:22,450 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 with policy now +2025-01-06 22:58:22,451 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:58:22,452 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 22:58:22,452 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:58:22,452 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:58:22,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:22,717 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/la2621ki-media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 +2025-01-06 22:58:23,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 22:58:23,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 +2025-01-06 22:58:23,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 22:58:23,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos +2025-01-06 22:58:23,244 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media +2025-01-06 22:58:23,539 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/tqbrjbop-media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 +2025-01-06 22:58:23,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:24,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:24,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:25,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:26,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:27,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:27,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:28,245 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:28,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:29,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:30,245 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:30,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:31,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:32,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:58:32,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:58:32,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:32,790 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:33,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:34,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:35,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:36,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:37,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:38,467 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:38,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:39,249 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:58:39,490 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/13k8zo9y-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:58:39,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:40,249 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:40,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:41,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:42,249 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:42,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:43,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:43,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:44,250 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:44,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:45,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:46,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:47,298 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 22:58:47,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:58:47,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:58:47,641 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:48,252 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:48,641 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:48,781 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:49,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:50,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:51,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:52,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:53,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:54,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:54,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:55,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:56,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:57,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:58,643 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:59,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:59,643 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:00,643 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:01,643 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:02,256 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:02,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:59:02,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:59:02,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:03,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:04,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:05,005 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:05,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:06,257 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:06,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:07,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:08,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:09,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:10,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:10,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:11,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:12,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:13,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:14,260 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:14,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:15,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:15,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:16,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:17,298 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 22:59:17,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:59:17,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:59:17,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:18,261 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:18,288 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 22:59:18,288 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 with policy now +2025-01-06 22:59:18,293 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:59:18,293 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 22:59:18,293 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:59:18,298 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:59:18,557 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/awrjwd0o-media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 +2025-01-06 22:59:18,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:19,261 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 22:59:19,261 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 +2025-01-06 22:59:19,262 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 22:59:19,601 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/ibe20cqo-media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 +2025-01-06 22:59:19,648 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:20,262 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:20,651 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:21,262 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:21,319 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:21,665 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:22,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:23,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:24,263 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:24,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:25,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:26,264 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:26,437 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:26,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:27,264 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:59:27,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:28,070 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 22:59:28,070 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:59:28,070 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:59:28,264 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 22:59:28,265 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:28,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:29,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:30,265 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:30,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:31,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:31,981 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:32,266 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:32,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:59:32,631 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:59:32,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:33,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:34,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:35,651 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:36,651 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:37,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:37,651 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:38,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:39,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:40,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:41,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:42,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:42,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:43,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:44,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:45,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:46,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:47,299 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 22:59:47,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:59:47,631 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:59:47,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:47,729 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:48,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:49,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:50,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:51,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:52,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:53,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:53,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:54,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:55,654 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:56,274 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:56,654 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:57,655 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:58,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:58,655 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:59,655 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:00,655 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:01,656 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:02,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:00:02,631 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:00:02,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:03,656 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:03,748 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:04,656 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:05,658 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:06,662 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:06,673 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:00:06,673 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 with policy now +2025-01-06 23:00:06,717 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:00:06,718 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:00:06,718 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:00:06,728 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:00:06,944 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/8t3lkz7m-media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 +2025-01-06 23:00:07,277 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:00:07,277 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 +2025-01-06 23:00:07,277 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 23:00:07,565 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/3y3lkwq7-media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 +2025-01-06 23:00:07,662 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:08,277 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:08,662 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:09,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:09,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:10,667 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:11,669 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:12,279 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:12,667 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:13,665 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:14,279 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:14,666 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:15,237 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:15,670 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:16,670 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:17,299 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:00:17,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:00:17,638 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:00:17,681 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:17,901 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:00:17,901 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:00:17,901 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:00:18,281 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:00:18,281 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:00:18,545 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/cheei9vj-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:00:18,670 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:19,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:20,281 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:20,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:20,741 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:21,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:22,282 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:22,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:23,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:24,283 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:24,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:25,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:25,838 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:26,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:27,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:28,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:29,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:30,285 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:30,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:31,317 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:31,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:32,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:00:32,632 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:00:32,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:33,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:34,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:35,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:36,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:36,717 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:37,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:38,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:39,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:40,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:41,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:42,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:42,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:43,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:44,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:45,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:46,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:47,300 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:00:47,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:00:47,633 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:00:47,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:47,789 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:48,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:49,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:50,675 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:51,676 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:52,292 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:52,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:53,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:53,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:54,676 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:55,676 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:56,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:57,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:58,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:58,510 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:00:58,510 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 with policy now +2025-01-06 23:00:58,515 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:00:58,515 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:00:58,516 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:00:58,527 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:00:58,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:58,767 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/ztlqoqfp-media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 +2025-01-06 23:00:59,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:00:59,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 +2025-01-06 23:00:59,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 23:00:59,553 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:59,582 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/q4b0n1a9-media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 +2025-01-06 23:00:59,683 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:00,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:00,679 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:01,678 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:02,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:01:02,640 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:01:02,685 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:03,684 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:04,297 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:04,686 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:04,811 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:05,688 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:06,297 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:06,689 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:07,699 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:08,698 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:09,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:10,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:10,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:11,699 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:12,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:13,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:14,300 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:01:14,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:14,966 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:01:14,966 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:01:14,967 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:01:15,300 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:01:15,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:16,301 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:16,313 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:16,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:17,300 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:01:17,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:01:17,636 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:01:17,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:18,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:19,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:20,302 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:20,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:21,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:22,275 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:22,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:23,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:24,304 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:24,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:25,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:26,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:27,641 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:27,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:28,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:29,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:30,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:31,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:32,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:01:32,634 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:01:32,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:32,758 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:33,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:34,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:35,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:36,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:37,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:38,641 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:38,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:39,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:40,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:41,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:42,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:43,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:43,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:44,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:45,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:46,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:47,300 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:01:47,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:01:47,634 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:01:47,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:48,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:48,794 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:49,706 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:50,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:51,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:52,706 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:53,722 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:54,209 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:54,314 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:54,354 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:01:54,355 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 with policy now +2025-01-06 23:01:54,364 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:01:54,365 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:01:54,365 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:01:54,371 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:01:54,618 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/8isq0r7b-media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 +2025-01-06 23:01:54,725 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:55,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:01:55,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 +2025-01-06 23:01:55,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 23:01:55,600 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/ql40hy80-media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 +2025-01-06 23:01:55,729 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:56,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:56,735 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:57,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:57,729 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:58,724 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:59,658 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:59,722 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:00,316 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:00,733 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:01,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:02,317 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:02,648 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:02:02,648 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:02:02,777 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:03,025 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:02:03,026 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:03,026 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:03,202 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 23:02:03,202 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval +2025-01-06 23:02:03,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table +2025-01-06 23:02:03,434 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:02:03,519 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/1w9ag9fk-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 23:02:03,554 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/pcgi9dpu-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:03,680 INFO wandb-upload_2:2132474 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphl6xy5fo +2025-01-06 23:02:03,725 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:03,852 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 23:02:03,863 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:02:04,077 INFO SenderThread:2132474 [sender.py:send_request_log_artifact():1525] logged artifact run-3fs5lhth-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjY2OTc1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc3MTAxMA==', 'latestArtifact': None}} +2025-01-06 23:02:04,077 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:02:04,077 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2025-01-06 23:02:04,078 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:02:04,078 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,079 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,079 DEBUG SenderThread:2132474 [sender.py:send():379] send: telemetry +2025-01-06 23:02:04,079 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,079 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,080 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,080 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,080 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,080 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,081 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,081 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,081 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,082 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,083 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,083 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,083 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,084 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,084 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,084 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,084 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,085 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,085 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,085 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,085 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,086 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,086 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 23:02:04,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:04,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:04,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:04,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json +2025-01-06 23:02:04,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval +2025-01-06 23:02:04,327 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/jtm3phv8-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 23:02:04,483 INFO wandb-upload_2:2132474 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyd6j289m +2025-01-06 23:02:04,499 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/0q3c0k8h-media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json +2025-01-06 23:02:04,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:04,865 INFO SenderThread:2132474 [sender.py:send_request_log_artifact():1525] logged artifact run-3fs5lhth-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjY2OTc5OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc3MTAxNQ==', 'latestArtifact': None}} +2025-01-06 23:02:04,865 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:02:04,865 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json with policy now +2025-01-06 23:02:04,866 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:02:04,866 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:02:04,866 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,867 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:05,134 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/qn8om7sk-media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json +2025-01-06 23:02:05,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:05,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:06,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:06,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:07,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:08,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:09,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:10,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:02:10,727 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:11,321 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml +2025-01-06 23:02:11,727 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:12,731 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:13,727 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:13,865 DEBUG SenderThread:2132474 [sender.py:send():379] send: exit +2025-01-06 23:02:13,865 INFO SenderThread:2132474 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 23:02:13,866 INFO SenderThread:2132474 [sender.py:send_exit():588] handling runtime: 296 +2025-01-06 23:02:13,867 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:13,867 INFO SenderThread:2132474 [sender.py:send_exit():594] send defer +2025-01-06 23:02:13,867 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,867 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 23:02:13,867 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:13,867 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 23:02:13,867 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 1 +2025-01-06 23:02:13,867 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,867 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 23:02:13,867 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:13,868 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 23:02:13,868 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 2 +2025-01-06 23:02:13,868 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,868 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 23:02:13,868 INFO HandlerThread:2132474 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 23:02:13,868 DEBUG SystemMonitor:2132474 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 23:02:13,868 DEBUG SystemMonitor:2132474 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 23:02:13,869 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 23:02:13,869 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 23:02:13,920 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 23:02:13,920 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 23:02:13,921 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined network monitor +2025-01-06 23:02:13,921 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:13,921 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 23:02:13,921 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 3 +2025-01-06 23:02:13,921 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:02:13,921 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,921 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 23:02:13,921 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:13,922 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 23:02:13,922 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 4 +2025-01-06 23:02:13,922 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,922 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 23:02:14,322 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:16,323 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:20,032 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:20,032 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:02:20,032 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 23:02:20,034 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,034 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 23:02:20,034 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 5 +2025-01-06 23:02:20,034 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:20,034 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,034 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 23:02:20,034 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:02:20,035 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:20,035 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:20,035 DEBUG SenderThread:2132474 [sender.py:send():379] send: summary +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:20,036 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 6 +2025-01-06 23:02:20,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,036 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 23:02:20,036 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 7 +2025-01-06 23:02:20,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:02:20,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,036 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 23:02:20,037 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,037 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 23:02:20,325 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:20,487 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 8 +2025-01-06 23:02:20,487 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,488 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 23:02:20,488 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,488 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 23:02:20,488 INFO SenderThread:2132474 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 23:02:20,490 INFO SenderThread:2132474 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 23:02:20,492 INFO SenderThread:2132474 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 23:02:20,506 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 9 +2025-01-06 23:02:20,506 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,506 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 23:02:20,507 DEBUG SenderThread:2132474 [sender.py:send():379] send: artifact +2025-01-06 23:02:20,734 INFO SenderThread:2132474 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjY1NzQxMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjY1NzQxMQ==', 'versionIndex': 430}}} +2025-01-06 23:02:20,734 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,734 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 23:02:20,734 INFO SenderThread:2132474 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 23:02:20,867 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:21,325 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:21,326 INFO SenderThread:2132474 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files +2025-01-06 23:02:21,326 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml config.yaml +2025-01-06 23:02:21,326 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json wandb-summary.json +2025-01-06 23:02:21,328 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/requirements.txt requirements.txt +2025-01-06 23:02:21,328 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-metadata.json wandb-metadata.json +2025-01-06 23:02:21,328 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:21,330 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log output.log +2025-01-06 23:02:21,330 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/diff.patch diff.patch +2025-01-06 23:02:21,331 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 10 +2025-01-06 23:02:21,332 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:21,333 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:21,334 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 23:02:21,335 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:21,336 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 23:02:21,336 INFO SenderThread:2132474 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 23:02:21,493 INFO wandb-upload_2:2132474 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/requirements.txt +2025-01-06 23:02:21,499 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:21,548 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml +2025-01-06 23:02:21,583 INFO wandb-upload_4:2132474 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:21,608 INFO wandb-upload_3:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/pqyy08m5-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:21,809 INFO Thread-11 (_thread_body):2132474 [sender.py:transition_state():614] send defer: 11 +2025-01-06 23:02:21,809 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:21,809 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 23:02:21,809 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:21,809 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 23:02:21,809 INFO SenderThread:2132474 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 23:02:21,811 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 12 +2025-01-06 23:02:21,811 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:21,811 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 23:02:21,811 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:21,811 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 23:02:21,811 INFO SenderThread:2132474 [file_stream.py:finish():601] file stream finish called +2025-01-06 23:02:21,867 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:22,034 INFO SenderThread:2132474 [file_stream.py:finish():605] file stream finish is done +2025-01-06 23:02:22,034 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 13 +2025-01-06 23:02:22,034 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:22,034 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:22,034 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 23:02:22,034 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:22,034 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 23:02:22,035 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 14 +2025-01-06 23:02:22,035 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:22,035 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 23:02:22,035 DEBUG SenderThread:2132474 [sender.py:send():379] send: final +2025-01-06 23:02:22,035 DEBUG SenderThread:2132474 [sender.py:send():379] send: footer +2025-01-06 23:02:22,035 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:22,035 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 23:02:22,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:22,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 23:02:22,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:22,036 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:22,036 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: server_info +2025-01-06 23:02:22,037 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 23:02:22,038 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 23:02:22,039 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:22,098 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:22,098 INFO MainThread:2132474 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 23:02:22,098 INFO MainThread:2132474 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 23:02:22,099 INFO MainThread:2132474 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 23:02:22,099 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 23:02:22,099 INFO HandlerThread:2132474 [handler.py:finish():882] shutting down handler +2025-01-06 23:02:23,037 INFO WriterThread:2132474 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/run-3fs5lhth.wandb +2025-01-06 23:02:23,098 INFO SenderThread:2132474 [sender.py:finish():1615] shutting down sender +2025-01-06 23:02:23,098 INFO SenderThread:2132474 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 23:02:23,098 INFO SenderThread:2132474 [file_pusher.py:join():175] waiting for file pusher diff --git a/3fs5lhth/wandb/debug.log b/3fs5lhth/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..626d88a56f4b7befa93d7aec21e4c33ba7bdd741 --- /dev/null +++ b/3fs5lhth/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Configure stats pid to 2132300 +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/settings +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:57:16,949 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:57:16,949 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:57:16,949 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:57:16,950 INFO MainThread:2132300 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug.log +2025-01-06 22:57:16,950 INFO MainThread:2132300 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug-internal.log +2025-01-06 22:57:16,950 INFO MainThread:2132300 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:57:16,950 INFO MainThread:2132300 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth'} +2025-01-06 22:57:16,951 INFO MainThread:2132300 [wandb_init.py:init():619] starting backend +2025-01-06 22:57:16,951 INFO MainThread:2132300 [wandb_init.py:init():623] setting up manager +2025-01-06 22:57:16,955 INFO MainThread:2132300 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:57:16,956 INFO MainThread:2132300 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:57:16,968 INFO MainThread:2132300 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:57:16,974 INFO MainThread:2132300 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:57:17,241 INFO MainThread:2132300 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:57:17,283 INFO MainThread:2132300 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:57:17,284 INFO MainThread:2132300 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:57:17,629 INFO MainThread:2132300 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:57:17,629 INFO MainThread:2132300 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:57:17,629 INFO MainThread:2132300 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:57:17,630 INFO MainThread:2132300 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:57:17,631 INFO MainThread:2132300 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:57:17,631 INFO MainThread:2132300 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/tb, True +2025-01-06 22:57:28,051 INFO MainThread:2132300 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 23:02:23,100 WARNING MsgRouterThr:2132300 [router.py:message_loop():77] message_loop has been closed diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code/FISH/eval_robot.py b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84f17b83a3b65229e8d9036a0c4dec47b2b88899 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736222236 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/diff.patch b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..9ddded5e048a5bd9c953b7e7d76c4e2599da0cac --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..a5e82a4 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..cc7e5e91447669fe989ab08b6e5b18934b0ea7ca --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.4, 0.04400911936343024, 0.8621948956718848, 0.05324718365067565, 0.803534984527916, 0.11712844677476308, 0.7487166672447589, 0.23282015328982517, 0.6637700199753331]]} \ No newline at end of file diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json new file mode 100644 index 0000000000000000000000000000000000000000..1aba730cd6ff0098d8e34f77cec3a1ed3ec76cc9 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[26, 30, 0.8666666666666667, 0.6702457964079827, 0.9674143417734872, 0.7176162749093653, 0.9618635511964704, 0.7440047487128529, 0.9417666866491841, 0.7574754058799775, 0.9385314005481322]]} \ No newline at end of file diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea2b57b7e1ebcdd4bccedeb41ea80b0ab9f89d89 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b48560eafc6e2046c0d3c86150f45f655df98b25eb482e7512147ab646dd46e +size 1518923 diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0688555e43b42137c1749d8d0eee120af55c2547 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:487641939ef48b883b67f94fbb0868673b10eff8a02668d21495242e2d202682 +size 1522323 diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..548084be99b60754939ddbf1baeb777bc278aa3d --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e00f3422b92e36dcbe3325b249715f22f7c8759ca190ee943a66de4faad718b8 +size 1497002 diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f20423d600c4b66874fccc8a60c778394a701604 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebb2a03c3b35f1d334353e9bc4e14c41ed6a57812dc8d683059e0f5ba1d2244c +size 1471298 diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d44df9795a46b58885ce5c5375a527929d8f01b1 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e244931479443b8b04bff9562fb1d38e7dd251741cb1dbeaa5f6e6ae33c06fc7 +size 1544494 diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..444f7217a1231e27ebe16b0b1a6d5cad76b6d811 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736222248.052863]: resetting environment +[INFO] [1736222248.057661]: cleared current plan +[INFO] [1736222248.058635]: moving to home +[INFO] [1736222249.061189]: reached home +[INFO] [1736222249.062481]: reset action history +[INFO] [1736222251.773432]: environment reset +Starting episode 0 +[INFO] [1736222251.777756]: resetting environment +[INFO] [1736222251.785227]: cleared current plan +[INFO] [1736222251.786117]: moving to home +[INFO] [1736222252.788674]: reached home +[INFO] [1736222252.790221]: reset action history +[INFO] [1736222255.549373]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736222262.867915]: resetting environment +[INFO] [1736222262.875106]: cleared current plan +[INFO] [1736222262.876200]: moving to home +[INFO] [1736222263.877371]: reached home +[INFO] [1736222263.878919]: reset action history +[INFO] [1736222266.587165]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/episode_rosbags/episode_0_2025-01-06-22-57-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2132300.log +started roslaunch server http://158.130.50.37:45115/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2132603] +process[print_text-2]: started with pid [2132604] +process[rosbag_record-3]: started with pid [2132605] +started bagging! +For topic gripper_width: timestamp difference is 100415110 for nearest: 1736222292357378601 - target: 1736222292256963491 at idx 249 +For topic gripper_width: timestamp difference is 102753057 for nearest: 1736222295257377281 - target: 1736222295154624224 at idx 249 +[INFO] [1736222301.472795]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736222302.242923]: Stored episode 1. +[INFO] [1736222302.244477]: Saving video... +[INFO] [1736222302.439958]: Video saved! +Starting episode 1 +[INFO] [1736222302.454041]: resetting environment +[INFO] [1736222302.462035]: cleared current plan +[INFO] [1736222302.462848]: moving to home +[INFO] [1736222305.769254]: reached home +[INFO] [1736222305.769719]: reset action history +[INFO] [1736222308.478192]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736222318.470896]: resetting environment +[INFO] [1736222318.474720]: cleared current plan +[INFO] [1736222318.474900]: moving to home +[INFO] [1736222319.475439]: reached home +[INFO] [1736222319.476286]: reset action history +[INFO] [1736222322.183926]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/episode_rosbags/episode_1_2025-01-06-22-58-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2132300.log +started roslaunch server http://158.130.50.37:33929/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2132731] +process[print_text-5]: started with pid [2132732] +process[rosbag_record-6]: started with pid [2132733] +started bagging! +For topic gripper_width: timestamp difference is -115380736 for nearest: 1736222326590708045 - target: 1736222326706088781 at idx 57 +For topic gripper_width: timestamp difference is 106687235 for nearest: 1736222341357402252 - target: 1736222341250715017 at idx 249 +For topic gripper_width: timestamp difference is 108810697 for nearest: 1736222343957376752 - target: 1736222343848566055 at idx 249 +For topic gripper_width: timestamp difference is 115442264 for nearest: 1736222352257375705 - target: 1736222352141933441 at idx 249 +[INFO] [1736222356.950727]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736222357.736038]: Stored episode 2. +[INFO] [1736222357.737038]: Saving video... +[INFO] [1736222358.255309]: Video saved! +Starting episode 2 +[INFO] [1736222358.300797]: resetting environment +[INFO] [1736222358.315153]: cleared current plan +[INFO] [1736222358.317722]: moving to home +[WARN] [1736222358.305532]: Plan exhausted +[INFO] [1736222361.622451]: reached home +[INFO] [1736222361.623585]: reset action history +[INFO] [1736222364.390730]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736222366.990652]: resetting environment +[INFO] [1736222367.001291]: cleared current plan +[INFO] [1736222367.001568]: moving to home +[INFO] [1736222368.003382]: reached home +[INFO] [1736222368.004087]: reset action history +[INFO] [1736222370.719429]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/episode_rosbags/episode_2_2025-01-06-22-59-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2132300.log +started roslaunch server http://158.130.50.37:39771/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2132874] +process[print_text-8]: started with pid [2132875] +process[rosbag_record-9]: started with pid [2132899] +started bagging! +For topic gripper_width: timestamp difference is -116800509 for nearest: 1736222394690714159 - target: 1736222394807514668 at idx 249 +[INFO] [1736222405.558885]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736222406.296252]: Stored episode 3. +[INFO] [1736222406.297352]: Saving video... +[INFO] [1736222406.639552]: Video saved! +Starting episode 3 +[INFO] [1736222406.680493]: resetting environment +[INFO] [1736222406.696279]: cleared current plan +[INFO] [1736222406.698044]: moving to home +[INFO] [1736222410.301896]: reached home +[INFO] [1736222410.305376]: reset action history +[INFO] [1736222413.195265]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736222417.669145]: resetting environment +[INFO] [1736222417.691352]: cleared current plan +[INFO] [1736222417.692019]: moving to home +[INFO] [1736222418.695400]: reached home +[INFO] [1736222418.697130]: reset action history +[INFO] [1736222421.423469]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/episode_rosbags/episode_3_2025-01-06-23-00-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2132300.log +started roslaunch server http://158.130.50.37:45729/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2133039] +process[print_text-11]: started with pid [2133040] +process[rosbag_record-12]: started with pid [2133041] +started bagging! +For topic gripper_width: timestamp difference is 110138155 for nearest: 1736222428257376171 - target: 1736222428147238016 at idx 86 +For topic gripper_width: timestamp difference is -105494809 for nearest: 1736222449890708852 - target: 1736222449996203661 at idx 249 +[INFO] [1736222457.399989]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736222458.141454]: Stored episode 4. +[INFO] [1736222458.147614]: Saving video... +[INFO] [1736222458.462468]: Video saved! +Starting episode 4 +[INFO] [1736222458.528286]: resetting environment +[INFO] [1736222458.550964]: cleared current plan +[INFO] [1736222458.552417]: moving to home +[INFO] [1736222462.155061]: reached home +[INFO] [1736222462.161212]: reset action history +[INFO] [1736222464.953107]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736222474.220058]: resetting environment +[INFO] [1736222474.256806]: cleared current plan +[INFO] [1736222474.260125]: moving to home +[INFO] [1736222475.269624]: reached home +[INFO] [1736222475.274547]: reset action history +[INFO] [1736222478.002243]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/episode_rosbags/episode_4_2025-01-06-23-01-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2132300.log +started roslaunch server http://158.130.50.37:44683/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2133158] +process[print_text-14]: started with pid [2133159] +process[rosbag_record-15]: started with pid [2133183] +started bagging! +For topic gripper_width: timestamp difference is -112093562 for nearest: 1736222482990707284 - target: 1736222483102800846 at idx 55 +[INFO] [1736222512.835092]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736222513.599057]: Stored episode 5. +[INFO] [1736222513.613150]: Saving video... +[INFO] [1736222514.260544]: Video saved! +[INFO] [1736222514.364800]: resetting environment +[INFO] [1736222514.397977]: cleared current plan +[INFO] [1736222514.407409]: moving to home +[WARN] [1736222514.208611]: Plan exhausted +[WARN] [1736222514.248782]: Plan exhausted +[WARN] [1736222514.291350]: Plan exhausted +[WARN] [1736222514.331246]: Plan exhausted +[WARN] [1736222514.368453]: Plan exhausted +[INFO] [1736222517.614906]: reached home +[INFO] [1736222517.617427]: reset action history +[INFO] [1736222520.413031]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/requirements.txt b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-metadata.json b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..58a16428fe8787f25837c573024f2c3287527237 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T03:57:17.306273", + "startedAt": "2025-01-07T03:57:16.946794", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.0555, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.908, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 638.5600509643555 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d99d8004ad307317ae3d76ca526fa3fadef876e9 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "0b48560eafc6e2046c0d3c86150f45f655df98b25eb482e7512147ab646dd46e", "size": 1518923, "path": "media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4"}, "global_step": 665, "_timestamp": 1736222523.7673342, "_runtime": 286.81069827079773, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "487641939ef48b883b67f94fbb0868673b10eff8a02668d21495242e2d202682", "size": 1522323, "path": "media/videos/eval/1_eval_1_487641939ef48b883b67.mp4"}, "eval/num_success": 2.0, "episode": 30.0, "eval/success_rate": 0.4000000059604645, "eval/success_rate_lb_0.01": 0.044009119272232056, "eval/success_rate_ub_0.01": 0.8621948957443237, "eval/success_rate_lb_0.025": 0.05324718356132507, "eval/success_rate_ub_0.025": 0.803534984588623, "eval/success_rate_lb_0.05": 0.11712844669818878, "eval/success_rate_ub_0.05": 0.7487166523933411, "eval/success_rate_lb_0.1": 0.23282015323638916, "eval/success_rate_ub_0.1": 0.6637700200080872, "eval/2_eval": {"_type": "video-file", "sha256": "e00f3422b92e36dcbe3325b249715f22f7c8759ca190ee943a66de4faad718b8", "size": 1497002, "path": "media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ebb2a03c3b35f1d334353e9bc4e14c41ed6a57812dc8d683059e0f5ba1d2244c", "size": 1471298, "path": "media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "e244931479443b8b04bff9562fb1d38e7dd251741cb1dbeaa5f6e6ae33c06fc7", "size": 1544494, "path": "media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "85c54431b4eb7c8145a72a6437cbffccb84a9cf38051f18b07dd0a638d459f25", "size": 401, "artifact_path": "wandb-client-artifact://qn6yedsr03wxdzdxjsx07gt1w6slsjguil7qwehhbk4r9u16ezkqmvigae1w5he3szsqg1lebi7egwlmsw6657epczazvz6c4t48v2b8okhjvpnn1gplmh79ga1cyysh/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://fqzdzcdgnvp49bln6x3umub8g4e2a8aydlpxqhj0wjzy2dx9jt8a1d5evqbi4d60zy6fmn15t3c7m95whtlp7aggh5549793289ebktf78n94q2se2gzi64otny0frcz:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 26, "total_num_episodes": 30, "total_success_rate": 0.8666666666666667, "total_success_rate_lb_0.01": 0.6702457964079827, "total_success_rate_ub_0.01": 0.9674143417734872, "total_success_rate_lb_0.025": 0.7176162749093653, "total_success_rate_ub_0.025": 0.9618635511964704, "total_success_rate_lb_0.05": 0.7440047487128529, "total_success_rate_ub_0.05": 0.9417666866491841, "total_success_rate_lb_0.1": 0.7574754058799775, "total_success_rate_ub_0.1": 0.9385314005481322, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "a90222fd6986d0de60ca1d9ceb25b68a395c4838b78bc467bfbbba3796af43a5", "size": 520, "artifact_path": "wandb-client-artifact://hujty64y3fb16jtsdfvtemw67w6e8dgh58dl0q5pl76k68h9ruq3m7gc3yn9n02wtuqvw2mxoubx4w0xrhlp7msjvqzhulhc26f43o1zk5ewkbrgrqoikjrrn6fy2au1/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://x3qzl8oz9excl016xuotxg3rtv1wabx14fzei3gblfvekvmrv3rytvox7a8fd79dfv9ujrpizjksna8nf703t8bat8egaxraka4au8ew6kl9v65q39ghkhgy06642a8o:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 296}, "eval/total_success_rate": 0.8666666746139526, "eval/total_success_rate_lb_0.01": 0.6702457666397095, "eval/total_success_rate_ub_0.01": 0.9674143195152283, "eval/total_success_rate_lb_0.025": 0.7176162600517273, "eval/total_success_rate_ub_0.025": 0.9618635773658752, "eval/total_success_rate_lb_0.05": 0.7440047264099121, "eval/total_success_rate_ub_0.05": 0.9417666792869568, "eval/total_success_rate_lb_0.1": 0.7574753761291504, "eval/total_success_rate_ub_0.1": 0.9385313987731934} \ No newline at end of file diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug-internal.log b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..595c8a55fc5993644a0ca22639ca25f9c06ababa --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug-internal.log @@ -0,0 +1,813 @@ +2025-01-06 22:57:16,956 INFO StreamThr :2132474 [internal.py:wandb_internal():85] W&B internal server running at pid: 2132474, started at: 2025-01-06 22:57:16.956330 +2025-01-06 22:57:16,957 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:57:16,969 INFO WriterThread:2132474 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/run-3fs5lhth.wandb +2025-01-06 22:57:16,969 DEBUG SenderThread:2132474 [sender.py:send():379] send: header +2025-01-06 22:57:16,976 DEBUG SenderThread:2132474 [sender.py:send():379] send: run +2025-01-06 22:57:17,225 INFO SenderThread:2132474 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files +2025-01-06 22:57:17,225 INFO SenderThread:2132474 [sender.py:_start_run_threads():1188] run started: 3fs5lhth with start time 1736222236.956636 +2025-01-06 22:57:17,241 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:57:17,242 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:57:17,288 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:57:17,293 DEBUG HandlerThread:2132474 [system_info.py:__init__():26] System info init +2025-01-06 22:57:17,293 DEBUG HandlerThread:2132474 [system_info.py:__init__():41] System info init done +2025-01-06 22:57:17,293 INFO HandlerThread:2132474 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:57:17,293 INFO SystemMonitor:2132474 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:57:17,293 INFO HandlerThread:2132474 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:57:17,293 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:57:17,294 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:57:17,295 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:57:17,295 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:57:17,296 INFO SystemMonitor:2132474 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:57:17,306 DEBUG HandlerThread:2132474 [system_info.py:probe():152] Probing system +2025-01-06 22:57:17,307 DEBUG HandlerThread:2132474 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:57:17,312 DEBUG HandlerThread:2132474 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:57:17,312 DEBUG HandlerThread:2132474 [system_info.py:probe():200] Probing system done +2025-01-06 22:57:17,312 DEBUG HandlerThread:2132474 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:57:17.306273', 'startedAt': '2025-01-07T03:57:16.946794', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.0555, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.908, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 638.5600509643555}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:57:17,312 INFO HandlerThread:2132474 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:57:17,312 INFO HandlerThread:2132474 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:57:17,312 DEBUG HandlerThread:2132474 [system_info.py:_save_code():44] Saving code +2025-01-06 22:57:17,313 DEBUG HandlerThread:2132474 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:57:17,313 DEBUG HandlerThread:2132474 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:57:17,377 DEBUG HandlerThread:2132474 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:57:17,377 INFO HandlerThread:2132474 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:57:17,380 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 22:57:17,380 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:57:17,381 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:57:17,381 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:57:17,627 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/39oobfg6-wandb-metadata.json +2025-01-06 22:57:17,629 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:57:17,629 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:57:17,629 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:57:17,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:57:17,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:17,631 INFO HandlerThread:2132474 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/tb" + save: true +} +control { + relay_id: "bc2d8352641d45d99d74edbb889444b7" +} +_info { + stream_id: "3fs5lhth" +} + +2025-01-06 22:57:17,632 DEBUG HandlerThread:2132474 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:57:17,649 INFO wandb-upload_2:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/kv49xzv5-diff.patch +2025-01-06 22:57:17,652 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/639feo2f-code/FISH/eval_robot.py +2025-01-06 22:57:17,762 DEBUG SenderThread:2132474 [sender.py:send():379] send: telemetry +2025-01-06 22:57:17,763 DEBUG SenderThread:2132474 [sender.py:send():379] send: tbrecord +2025-01-06 22:57:17,763 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 22:57:17,763 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 with policy live +2025-01-06 22:57:17,951 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/hr5aukav-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:57:18,226 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/requirements.txt +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code/FISH/eval_robot.py +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-metadata.json +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/diff.patch +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code +2025-01-06 22:57:18,227 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code/FISH +2025-01-06 22:57:18,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:19,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:20,226 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:20,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:21,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:22,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:22,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:23,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:24,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:25,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:26,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:27,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:27,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:28,052 DEBUG SenderThread:2132474 [sender.py:send():379] send: config +2025-01-06 22:57:28,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:29,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:30,228 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:30,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:31,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:32,629 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:57:32,629 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:57:32,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:32,685 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:33,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:34,229 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:34,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:35,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:36,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:37,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:38,231 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:38,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:38,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:39,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:40,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:41,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:42,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:43,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:43,877 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:44,232 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:44,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:45,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:46,233 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:46,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:47,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:57:47,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:57:47,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:48,234 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:57:48,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:49,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:49,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:50,234 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml +2025-01-06 22:57:50,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:51,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:52,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:53,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:54,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:55,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:57:55,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:56,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:57,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:58,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:57:59,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:00,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:00,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:01,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:02,629 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:58:02,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:58:02,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:03,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:04,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:05,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:05,747 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:06,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:07,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:08,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:09,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:10,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:11,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:11,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:12,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:13,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:14,241 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:14,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:15,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:16,241 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:16,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:17,296 DEBUG SystemMonitor:2132474 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:58:17,297 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 22:58:17,298 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:17,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:58:17,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:58:17,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:18,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:19,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:20,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:21,637 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:22,440 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:22,450 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 22:58:22,450 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 with policy now +2025-01-06 22:58:22,451 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:58:22,452 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 22:58:22,452 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:58:22,452 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:58:22,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:22,717 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/la2621ki-media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 +2025-01-06 22:58:23,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 22:58:23,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 +2025-01-06 22:58:23,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 22:58:23,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos +2025-01-06 22:58:23,244 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media +2025-01-06 22:58:23,539 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/tqbrjbop-media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 +2025-01-06 22:58:23,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:24,243 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:24,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:25,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:26,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:27,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:27,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:28,245 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:28,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:29,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:30,245 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:30,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:31,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:32,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:58:32,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:58:32,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:32,790 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:33,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:34,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:35,639 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:36,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:37,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:38,467 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:38,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:39,249 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:58:39,490 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/13k8zo9y-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:58:39,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:40,249 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:40,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:41,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:42,249 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:42,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:43,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:43,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:44,250 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:44,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:45,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:46,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:47,298 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 22:58:47,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:58:47,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:58:47,641 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:48,252 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:58:48,641 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:48,781 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:49,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:50,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:51,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:52,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:53,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:54,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:54,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:55,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:56,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:57,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:58,643 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:58:59,633 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:58:59,643 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:00,643 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:01,643 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:02,256 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:02,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:59:02,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:59:02,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:03,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:04,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:05,005 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:05,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:06,257 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:06,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:07,644 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:08,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:09,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:10,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:10,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:11,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:12,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:13,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:14,260 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:14,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:15,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:15,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:16,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:17,298 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 22:59:17,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:59:17,630 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:59:17,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:18,261 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:18,288 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 22:59:18,288 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 with policy now +2025-01-06 22:59:18,293 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:59:18,293 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 22:59:18,293 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:59:18,298 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:59:18,557 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/awrjwd0o-media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 +2025-01-06 22:59:18,646 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:19,261 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 22:59:19,261 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 +2025-01-06 22:59:19,262 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 22:59:19,601 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/ibe20cqo-media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 +2025-01-06 22:59:19,648 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:20,262 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:20,651 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:21,262 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:21,319 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:21,665 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:22,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:23,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:24,263 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:24,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:25,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:26,264 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:26,437 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:26,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:27,264 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 22:59:27,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:28,070 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 22:59:28,070 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:59:28,070 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:59:28,264 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 22:59:28,265 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:28,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:29,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:30,265 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:30,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:31,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:31,981 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:32,266 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:32,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:59:32,631 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:59:32,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:33,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:34,650 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:35,651 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:36,651 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:37,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:37,651 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:38,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:39,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:40,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:41,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:42,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:42,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:43,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:44,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:45,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:46,652 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:47,299 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 22:59:47,630 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:59:47,631 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:59:47,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:47,729 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:48,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:49,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:50,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:51,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:52,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:53,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:53,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:54,653 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:55,654 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:56,274 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 22:59:56,654 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:57,655 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:58,635 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:59:58,655 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:59:59,655 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:00,655 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:01,656 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:02,631 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:00:02,631 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:00:02,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:03,656 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:03,748 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:04,656 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:05,658 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:06,662 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:06,673 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:00:06,673 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 with policy now +2025-01-06 23:00:06,717 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:00:06,718 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:00:06,718 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:00:06,728 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:00:06,944 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/8t3lkz7m-media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 +2025-01-06 23:00:07,277 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:00:07,277 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 +2025-01-06 23:00:07,277 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 23:00:07,565 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/3y3lkwq7-media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 +2025-01-06 23:00:07,662 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:08,277 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:08,662 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:09,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:09,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:10,667 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:11,669 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:12,279 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:12,667 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:13,665 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:14,279 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:14,666 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:15,237 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:15,670 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:16,670 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:17,299 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:00:17,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:00:17,638 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:00:17,681 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:17,901 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:00:17,901 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:00:17,901 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:00:18,281 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:00:18,281 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:00:18,545 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/cheei9vj-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:00:18,670 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:19,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:20,281 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:20,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:20,741 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:21,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:22,282 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:22,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:23,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:24,283 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:24,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:25,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:25,838 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:26,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:27,671 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:28,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:29,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:30,285 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:30,672 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:31,317 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:31,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:32,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:00:32,632 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:00:32,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:33,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:34,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:35,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:36,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:36,717 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:37,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:38,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:39,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:40,673 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:41,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:42,638 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:42,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:43,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:44,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:45,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:46,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:47,300 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:00:47,632 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:00:47,633 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:00:47,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:47,789 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:48,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:49,674 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:50,675 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:51,676 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:52,292 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:52,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:53,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:53,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:54,676 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:55,676 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:56,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:57,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:58,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:00:58,510 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:00:58,510 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 with policy now +2025-01-06 23:00:58,515 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:00:58,515 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:00:58,516 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:00:58,527 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:00:58,677 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:00:58,767 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/ztlqoqfp-media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 +2025-01-06 23:00:59,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:00:59,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 +2025-01-06 23:00:59,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 23:00:59,553 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:00:59,582 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/q4b0n1a9-media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 +2025-01-06 23:00:59,683 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:00,295 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:00,679 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:01,678 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:02,640 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:01:02,640 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:01:02,685 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:03,684 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:04,297 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:04,686 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:04,811 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:05,688 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:06,297 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:06,689 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:07,699 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:08,698 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:09,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:10,645 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:10,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:11,699 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:12,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:13,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:14,300 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:01:14,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:14,966 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:01:14,966 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:01:14,967 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:01:15,300 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:01:15,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:16,301 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:16,313 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:16,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:17,300 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:01:17,636 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:01:17,636 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:01:17,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:18,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:19,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:20,302 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:20,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:21,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:22,275 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:22,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:23,701 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:24,304 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:24,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:25,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:26,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:27,641 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:27,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:28,702 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:29,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:30,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:31,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:32,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:01:32,634 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:01:32,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:32,758 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:33,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:34,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:35,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:36,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:37,703 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:38,641 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:38,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:39,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:40,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:41,704 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:42,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:43,642 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:43,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:44,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:45,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:46,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:47,300 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:01:47,634 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:01:47,634 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:01:47,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:48,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:48,794 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:49,706 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:50,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:51,705 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:52,706 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:53,722 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:54,209 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:54,314 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:54,354 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:01:54,355 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 with policy now +2025-01-06 23:01:54,364 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:01:54,365 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:01:54,365 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:01:54,371 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:01:54,618 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/8isq0r7b-media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 +2025-01-06 23:01:54,725 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:55,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:01:55,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 +2025-01-06 23:01:55,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval +2025-01-06 23:01:55,600 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/ql40hy80-media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 +2025-01-06 23:01:55,729 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:56,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:56,735 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:57,315 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:01:57,729 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:58,724 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:01:59,658 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:01:59,722 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:00,316 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:00,733 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:01,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:02,317 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:02,648 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 23:02:02,648 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: stop_status +2025-01-06 23:02:02,777 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:03,025 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:02:03,026 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:03,026 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:03,202 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 23:02:03,202 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media +2025-01-06 23:02:03,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval +2025-01-06 23:02:03,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table +2025-01-06 23:02:03,434 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:02:03,519 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/1w9ag9fk-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 23:02:03,554 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/pcgi9dpu-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:03,680 INFO wandb-upload_2:2132474 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphl6xy5fo +2025-01-06 23:02:03,725 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:03,852 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 23:02:03,863 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 23:02:04,077 INFO SenderThread:2132474 [sender.py:send_request_log_artifact():1525] logged artifact run-3fs5lhth-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjY2OTc1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc3MTAxMA==', 'latestArtifact': None}} +2025-01-06 23:02:04,077 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:02:04,077 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2025-01-06 23:02:04,078 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:02:04,078 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,079 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,079 DEBUG SenderThread:2132474 [sender.py:send():379] send: telemetry +2025-01-06 23:02:04,079 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,079 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,080 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,080 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,080 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,080 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,081 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,081 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,081 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,082 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,083 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,083 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,083 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,084 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,084 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,084 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,084 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,085 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,085 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,085 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,085 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,086 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:04,086 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 23:02:04,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:04,318 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:04,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:04,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json +2025-01-06 23:02:04,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval +2025-01-06 23:02:04,327 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/jtm3phv8-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 23:02:04,483 INFO wandb-upload_2:2132474 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyd6j289m +2025-01-06 23:02:04,499 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/0q3c0k8h-media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json +2025-01-06 23:02:04,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:04,865 INFO SenderThread:2132474 [sender.py:send_request_log_artifact():1525] logged artifact run-3fs5lhth-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjY2OTc5OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc3MTAxNQ==', 'latestArtifact': None}} +2025-01-06 23:02:04,865 DEBUG SenderThread:2132474 [sender.py:send():379] send: files +2025-01-06 23:02:04,865 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json with policy now +2025-01-06 23:02:04,866 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:02:04,866 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:02:04,866 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:04,867 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:05,134 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/qn8om7sk-media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json +2025-01-06 23:02:05,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:05,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:06,319 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:06,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:07,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:08,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:09,726 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:10,649 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:02:10,727 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:11,321 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml +2025-01-06 23:02:11,727 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:12,731 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:13,727 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:13,865 DEBUG SenderThread:2132474 [sender.py:send():379] send: exit +2025-01-06 23:02:13,865 INFO SenderThread:2132474 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 23:02:13,866 INFO SenderThread:2132474 [sender.py:send_exit():588] handling runtime: 296 +2025-01-06 23:02:13,867 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:13,867 INFO SenderThread:2132474 [sender.py:send_exit():594] send defer +2025-01-06 23:02:13,867 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,867 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 23:02:13,867 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:13,867 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 23:02:13,867 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 1 +2025-01-06 23:02:13,867 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,867 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 23:02:13,867 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:13,868 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 23:02:13,868 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 2 +2025-01-06 23:02:13,868 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,868 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 23:02:13,868 INFO HandlerThread:2132474 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 23:02:13,868 DEBUG SystemMonitor:2132474 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 23:02:13,868 DEBUG SystemMonitor:2132474 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 23:02:13,869 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 23:02:13,869 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 23:02:13,920 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 23:02:13,920 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 23:02:13,921 INFO HandlerThread:2132474 [interfaces.py:finish():200] Joined network monitor +2025-01-06 23:02:13,921 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:13,921 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 23:02:13,921 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 3 +2025-01-06 23:02:13,921 DEBUG SenderThread:2132474 [sender.py:send():379] send: stats +2025-01-06 23:02:13,921 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,921 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 23:02:13,921 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:13,922 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 23:02:13,922 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 4 +2025-01-06 23:02:13,922 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:13,922 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 23:02:14,322 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:16,323 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:20,032 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:20,032 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:02:20,032 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 23:02:20,034 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,034 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 23:02:20,034 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 5 +2025-01-06 23:02:20,034 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:20,034 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,034 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 23:02:20,034 DEBUG SenderThread:2132474 [sender.py:send():379] send: history +2025-01-06 23:02:20,035 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: summary_record +2025-01-06 23:02:20,035 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:20,035 DEBUG SenderThread:2132474 [sender.py:send():379] send: summary +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 23:02:20,036 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 6 +2025-01-06 23:02:20,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,036 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 23:02:20,036 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 23:02:20,036 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 7 +2025-01-06 23:02:20,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 23:02:20,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,036 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 23:02:20,037 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,037 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 23:02:20,325 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:20,487 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 8 +2025-01-06 23:02:20,487 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,488 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 23:02:20,488 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,488 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 23:02:20,488 INFO SenderThread:2132474 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 23:02:20,490 INFO SenderThread:2132474 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 23:02:20,492 INFO SenderThread:2132474 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 23:02:20,506 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 9 +2025-01-06 23:02:20,506 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:20,506 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 23:02:20,507 DEBUG SenderThread:2132474 [sender.py:send():379] send: artifact +2025-01-06 23:02:20,734 INFO SenderThread:2132474 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjY1NzQxMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjY1NzQxMQ==', 'versionIndex': 430}}} +2025-01-06 23:02:20,734 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:20,734 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 23:02:20,734 INFO SenderThread:2132474 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 23:02:20,867 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:21,325 INFO Thread-12 :2132474 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:21,326 INFO SenderThread:2132474 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files +2025-01-06 23:02:21,326 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml config.yaml +2025-01-06 23:02:21,326 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json wandb-summary.json +2025-01-06 23:02:21,328 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/requirements.txt requirements.txt +2025-01-06 23:02:21,328 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-metadata.json wandb-metadata.json +2025-01-06 23:02:21,328 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:21,330 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log output.log +2025-01-06 23:02:21,330 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/diff.patch diff.patch +2025-01-06 23:02:21,331 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 media/videos/eval/3_eval_3_ebb2a03c3b35f1d33435.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 media/videos/eval/1_eval_1_487641939ef48b883b67.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 media/videos/eval/4_eval_4_e244931479443b8b04bf.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 media/videos/eval/2_eval_2_e00f3422b92e36dcbe33.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 media/videos/eval/0_eval_0_0b48560eafc6e2046c0d.mp4 +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json media/table/eval/total_success_rate_ci_6_a90222fd6986d0de60ca.table.json +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-06 23:02:21,332 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 10 +2025-01-06 23:02:21,332 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:21,333 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:21,334 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 23:02:21,335 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:21,336 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 23:02:21,336 INFO SenderThread:2132474 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 23:02:21,493 INFO wandb-upload_2:2132474 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/requirements.txt +2025-01-06 23:02:21,499 INFO wandb-upload_0:2132474 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/wandb-summary.json +2025-01-06 23:02:21,548 INFO wandb-upload_1:2132474 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/config.yaml +2025-01-06 23:02:21,583 INFO wandb-upload_4:2132474 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/files/output.log +2025-01-06 23:02:21,608 INFO wandb-upload_3:2132474 [upload_job.py:push():130] Uploaded file /tmp/tmpg5vamc1mwandb/pqyy08m5-events.out.tfevents.1736222237.leonmkim-ROG-Strix-G15CS-G15CS.2132300.0 +2025-01-06 23:02:21,809 INFO Thread-11 (_thread_body):2132474 [sender.py:transition_state():614] send defer: 11 +2025-01-06 23:02:21,809 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:21,809 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 23:02:21,809 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:21,809 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 23:02:21,809 INFO SenderThread:2132474 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 23:02:21,811 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 12 +2025-01-06 23:02:21,811 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:21,811 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 23:02:21,811 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:21,811 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 23:02:21,811 INFO SenderThread:2132474 [file_stream.py:finish():601] file stream finish called +2025-01-06 23:02:21,867 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:22,034 INFO SenderThread:2132474 [file_stream.py:finish():605] file stream finish is done +2025-01-06 23:02:22,034 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 13 +2025-01-06 23:02:22,034 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:22,034 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:22,034 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 23:02:22,034 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:22,034 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 23:02:22,035 INFO SenderThread:2132474 [sender.py:transition_state():614] send defer: 14 +2025-01-06 23:02:22,035 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: defer +2025-01-06 23:02:22,035 INFO HandlerThread:2132474 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 23:02:22,035 DEBUG SenderThread:2132474 [sender.py:send():379] send: final +2025-01-06 23:02:22,035 DEBUG SenderThread:2132474 [sender.py:send():379] send: footer +2025-01-06 23:02:22,035 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: defer +2025-01-06 23:02:22,035 INFO SenderThread:2132474 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 23:02:22,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:22,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 23:02:22,036 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 23:02:22,036 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:22,036 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: server_info +2025-01-06 23:02:22,037 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 23:02:22,038 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 23:02:22,039 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 23:02:22,098 DEBUG SenderThread:2132474 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 23:02:22,098 INFO MainThread:2132474 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 23:02:22,098 INFO MainThread:2132474 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 23:02:22,099 INFO MainThread:2132474 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 23:02:22,099 DEBUG HandlerThread:2132474 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 23:02:22,099 INFO HandlerThread:2132474 [handler.py:finish():882] shutting down handler +2025-01-06 23:02:23,037 INFO WriterThread:2132474 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/run-3fs5lhth.wandb +2025-01-06 23:02:23,098 INFO SenderThread:2132474 [sender.py:finish():1615] shutting down sender +2025-01-06 23:02:23,098 INFO SenderThread:2132474 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 23:02:23,098 INFO SenderThread:2132474 [file_pusher.py:join():175] waiting for file pusher diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug.log b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..626d88a56f4b7befa93d7aec21e4c33ba7bdd741 --- /dev/null +++ b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Configure stats pid to 2132300 +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/settings +2025-01-06 22:57:16,948 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:57:16,949 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:57:16,949 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:57:16,949 INFO MainThread:2132300 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:57:16,950 INFO MainThread:2132300 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug.log +2025-01-06 22:57:16,950 INFO MainThread:2132300 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/logs/debug-internal.log +2025-01-06 22:57:16,950 INFO MainThread:2132300 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:57:16,950 INFO MainThread:2132300 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth'} +2025-01-06 22:57:16,951 INFO MainThread:2132300 [wandb_init.py:init():619] starting backend +2025-01-06 22:57:16,951 INFO MainThread:2132300 [wandb_init.py:init():623] setting up manager +2025-01-06 22:57:16,955 INFO MainThread:2132300 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:57:16,956 INFO MainThread:2132300 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:57:16,968 INFO MainThread:2132300 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:57:16,974 INFO MainThread:2132300 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:57:17,241 INFO MainThread:2132300 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:57:17,283 INFO MainThread:2132300 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:57:17,284 INFO MainThread:2132300 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:57:17,629 INFO MainThread:2132300 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:57:17,629 INFO MainThread:2132300 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:57:17,629 INFO MainThread:2132300 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:57:17,630 INFO MainThread:2132300 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:57:17,631 INFO MainThread:2132300 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:57:17,631 INFO MainThread:2132300 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/3fs5lhth/tb, True +2025-01-06 22:57:28,051 INFO MainThread:2132300 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 23:02:23,100 WARNING MsgRouterThr:2132300 [router.py:message_loop():77] message_loop has been closed diff --git a/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/run-3fs5lhth.wandb b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/run-3fs5lhth.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a4cee3078068b03648ab1fa2338cee4a534eb1a0 Binary files /dev/null and b/3fs5lhth/wandb/run-20250106_225716-3fs5lhth/run-3fs5lhth.wandb differ diff --git a/bmooajlw/.hydra/config.yaml b/bmooajlw/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/bmooajlw/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/bmooajlw/.hydra/hydra.yaml b/bmooajlw/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dcab282e9b59994266efa20520ef728e524fecf --- /dev/null +++ b/bmooajlw/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/bmooajlw + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/bmooajlw/.hydra/overrides.yaml b/bmooajlw/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/bmooajlw/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/bmooajlw/eval_policy.log b/bmooajlw/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..6f318bd9c388bbc4ab78ef67112eb8f80dbcc918 --- /dev/null +++ b/bmooajlw/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 18:06:11,003][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 18:06:11,006][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 18:06:14,777][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2288afeb8b61244686c351b5577d53b6e17303d --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 1 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1017_0' +wandb_notes: 1017_0_restarted_11 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 5 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/kd9mh2vh/.hydra/config.yaml b/kd9mh2vh/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/kd9mh2vh/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/kd9mh2vh/.hydra/hydra.yaml b/kd9mh2vh/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97be0b9b7e1c10929a4cea00e91e7e1e0b4a7623 --- /dev/null +++ b/kd9mh2vh/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/kd9mh2vh/.hydra/overrides.yaml b/kd9mh2vh/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/kd9mh2vh/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/kd9mh2vh/episode_rosbags/aligned_depth_to_color_K.npy b/kd9mh2vh/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kd9mh2vh/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kd9mh2vh/episode_rosbags/cam_tf_world.npy b/kd9mh2vh/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/kd9mh2vh/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/kd9mh2vh/episode_rosbags/color_K.npy b/kd9mh2vh/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kd9mh2vh/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kd9mh2vh/episode_rosbags/depth_K.npy b/kd9mh2vh/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/kd9mh2vh/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/kd9mh2vh/eval_robot.log b/kd9mh2vh/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..b519fdfeb04effe92d86f94048d90fc7cb711304 --- /dev/null +++ b/kd9mh2vh/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 16:19:21,668][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 16:19:21,670][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/kd9mh2vh/eval_video/0_eval.mp4 b/kd9mh2vh/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57bdafb889d1c369058938333621155f869caeb4 --- /dev/null +++ b/kd9mh2vh/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89262cab025e888a6f21af0c8e0e45f248aec311e89fe6eff8c0a6420dd9edd7 +size 1537570 diff --git a/kd9mh2vh/eval_video/1_eval.mp4 b/kd9mh2vh/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c04447e082e0c64c4187aa6938a4350de9ad776e --- /dev/null +++ b/kd9mh2vh/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:887535ba826c760b568aab1c6a63fa73d25e9aedf01bed19177ee99551a8a746 +size 1531638 diff --git a/kd9mh2vh/eval_video/2_eval.mp4 b/kd9mh2vh/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c9a87e79e1d951c8acbb11ce27e64fcbaaacfeb1 --- /dev/null +++ b/kd9mh2vh/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de6316f5b26b7e9850a50c030889b874192ec1e76b280f906cc47e75c46cbbfb +size 1527940 diff --git a/kd9mh2vh/eval_video/3_eval.mp4 b/kd9mh2vh/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2abaa7737badc4afe11282c13b07a7e77e110b4b --- /dev/null +++ b/kd9mh2vh/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:295a37e1d2a179eff24306db755a7658dce0634a5554cc23e6d829e32b922edc +size 1544801 diff --git a/kd9mh2vh/eval_video/4_eval.mp4 b/kd9mh2vh/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..275cdd693d55cdfe56d93f61ee7a3e0c90b6f148 --- /dev/null +++ b/kd9mh2vh/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a7a2360b5bcbe4011d794f92c18f5dea65247cc82a1a66fedb61fcdde9575b +size 1548137 diff --git a/kd9mh2vh/tb/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 b/kd9mh2vh/tb/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 new file mode 100644 index 0000000000000000000000000000000000000000..1ad5aa6427bacb92946031e50d106df8b10c59f0 --- /dev/null +++ b/kd9mh2vh/tb/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e74f2eaa8003ea89e5065faebbcc695b858368da48270dc0bbf634d4468f688 +size 6539 diff --git a/kd9mh2vh/wandb/debug-internal.log b/kd9mh2vh/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a094fc542e475b6ef1312cf05b7773ff7a02ab08 --- /dev/null +++ b/kd9mh2vh/wandb/debug-internal.log @@ -0,0 +1,1069 @@ +2025-01-07 16:19:27,467 INFO StreamThr :2183860 [internal.py:wandb_internal():85] W&B internal server running at pid: 2183860, started at: 2025-01-07 16:19:27.466675 +2025-01-07 16:19:27,468 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status +2025-01-07 16:19:27,479 INFO WriterThread:2183860 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/run-kd9mh2vh.wandb +2025-01-07 16:19:27,480 DEBUG SenderThread:2183860 [sender.py:send():379] send: header +2025-01-07 16:19:27,490 DEBUG SenderThread:2183860 [sender.py:send():379] send: run +2025-01-07 16:19:27,771 INFO SenderThread:2183860 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files +2025-01-07 16:19:27,771 INFO SenderThread:2183860 [sender.py:_start_run_threads():1188] run started: kd9mh2vh with start time 1736284767.467768 +2025-01-07 16:19:27,787 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 16:19:27,787 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: check_version +2025-01-07 16:19:27,862 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 16:19:27,885 DEBUG HandlerThread:2183860 [system_info.py:__init__():26] System info init +2025-01-07 16:19:27,885 DEBUG HandlerThread:2183860 [system_info.py:__init__():41] System info init done +2025-01-07 16:19:27,885 INFO HandlerThread:2183860 [system_monitor.py:start():194] Starting system monitor +2025-01-07 16:19:27,885 INFO SystemMonitor:2183860 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 16:19:27,886 INFO HandlerThread:2183860 [system_monitor.py:probe():214] Collecting system info +2025-01-07 16:19:27,886 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 16:19:27,887 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started disk monitoring +2025-01-07 16:19:27,887 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 16:19:27,888 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started memory monitoring +2025-01-07 16:19:27,888 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started network monitoring +2025-01-07 16:19:27,913 DEBUG HandlerThread:2183860 [system_info.py:probe():152] Probing system +2025-01-07 16:19:27,914 DEBUG HandlerThread:2183860 [system_info.py:_probe_git():137] Probing git +2025-01-07 16:19:27,920 DEBUG HandlerThread:2183860 [system_info.py:_probe_git():145] Probing git done +2025-01-07 16:19:27,920 DEBUG HandlerThread:2183860 [system_info.py:probe():200] Probing system done +2025-01-07 16:19:27,920 DEBUG HandlerThread:2183860 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:19:27.913455', 'startedAt': '2025-01-07T21:19:27.458529', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.7955, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.992, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.029, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4502.343, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 650.7937316894531}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 16:19:27,920 INFO HandlerThread:2183860 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 16:19:27,920 INFO HandlerThread:2183860 [system_monitor.py:probe():227] Publishing system info +2025-01-07 16:19:27,920 DEBUG HandlerThread:2183860 [system_info.py:_save_code():44] Saving code +2025-01-07 16:19:27,922 DEBUG HandlerThread:2183860 [system_info.py:_save_code():67] Saving code done +2025-01-07 16:19:27,922 DEBUG HandlerThread:2183860 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 16:19:27,952 DEBUG HandlerThread:2183860 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 16:19:27,952 INFO HandlerThread:2183860 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 16:19:27,955 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:19:27,955 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 16:19:27,955 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 16:19:28,202 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/a5pvksfa-wandb-metadata.json +2025-01-07 16:19:28,208 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/p21bpd4i-code/FISH/eval_robot.py +2025-01-07 16:19:28,213 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 16:19:28,214 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: python_packages +2025-01-07 16:19:28,214 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:19:28,214 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:19:28,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:28,216 INFO HandlerThread:2183860 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/tb" + save: true +} +control { + relay_id: "fe15f45b5ac547a1abc47cd88a1a163b" +} +_info { + stream_id: "kd9mh2vh" +} + +2025-01-07 16:19:28,216 DEBUG HandlerThread:2183860 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 16:19:28,311 DEBUG SenderThread:2183860 [sender.py:send():379] send: telemetry +2025-01-07 16:19:28,311 DEBUG SenderThread:2183860 [sender.py:send():379] send: tbrecord +2025-01-07 16:19:28,311 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:19:28,311 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 with policy live +2025-01-07 16:19:28,485 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/wmziujzy-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:19:28,773 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:19:28,773 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code/FISH/eval_robot.py +2025-01-07 16:19:28,773 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-metadata.json +2025-01-07 16:19:28,773 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/requirements.txt +2025-01-07 16:19:28,774 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:19:28,774 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code/FISH +2025-01-07 16:19:28,774 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code +2025-01-07 16:19:29,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:30,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:30,772 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:19:31,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:32,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:33,214 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:33,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:34,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:35,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:36,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:37,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:38,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:38,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:39,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:40,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:41,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:42,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:43,214 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:19:43,214 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:19:43,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:43,281 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:44,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:45,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:46,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:47,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:48,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:49,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:49,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:50,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:51,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:52,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:53,105 DEBUG SenderThread:2183860 [sender.py:send():379] send: config +2025-01-07 16:19:53,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:54,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:54,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:54,779 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:19:55,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:56,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:56,779 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:19:57,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:58,214 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:19:58,214 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:19:58,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:59,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:59,348 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:59,780 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml +2025-01-07 16:20:00,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:00,780 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:20:01,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:02,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:03,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:04,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:04,781 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:20:05,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:05,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:06,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:07,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:08,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:09,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:10,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:10,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:11,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:12,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:13,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:20:13,215 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:20:13,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:14,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:15,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:15,314 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:16,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:17,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:18,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:19,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:20,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:21,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:21,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:22,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:23,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:24,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:25,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:26,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:26,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:27,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:27,889 DEBUG SystemMonitor:2183860 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 16:20:27,889 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:20:28,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:20:28,215 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:20:28,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:29,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:30,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:31,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:31,380 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:32,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:33,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:34,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:35,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:36,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:37,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:37,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:38,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:39,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:40,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:41,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:42,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:43,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:20:43,215 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:20:43,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:43,310 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:44,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:45,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:46,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:47,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:48,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:49,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:49,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:50,226 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:51,226 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:52,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:53,226 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:54,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:54,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:55,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:56,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:57,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:57,890 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:20:58,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:20:58,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:20:58,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:59,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:59,308 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:00,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:01,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:02,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:03,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:04,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:05,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:05,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:06,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:07,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:08,229 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:09,229 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:10,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:10,229 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:11,229 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:12,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:13,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:21:13,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:21:13,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:14,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:15,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:15,366 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:16,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:17,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:18,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:19,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:20,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:21,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:21,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:22,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:23,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:24,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:25,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:26,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:26,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:27,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:27,890 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:21:28,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:21:28,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:21:28,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:29,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:30,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:31,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:31,291 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:32,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:33,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:34,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:35,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:36,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:37,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:37,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:38,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:39,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:40,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:41,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:42,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:42,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:43,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:21:43,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:21:43,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:44,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:45,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:46,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:47,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:47,289 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:48,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:49,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:50,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:51,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:52,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:53,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:53,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:54,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:55,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:56,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:57,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:57,891 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:21:58,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:21:58,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:21:58,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:58,321 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:59,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:00,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:01,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:02,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:03,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:04,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:04,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:05,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:06,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:07,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:08,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:09,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:09,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:10,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:11,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:12,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:13,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:22:13,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:22:13,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:14,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:14,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:15,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:16,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:17,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:18,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:19,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:20,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:20,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:21,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:22,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:23,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:24,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:25,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:25,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:26,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:27,239 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:27,891 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:22:28,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:22:28,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:22:28,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:29,239 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:30,239 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:30,295 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:31,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:32,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:33,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:34,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:35,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:36,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:36,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:37,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:38,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:39,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:40,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:41,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:41,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:42,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:43,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:22:43,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:22:43,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:44,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:45,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:46,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:46,360 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:47,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:48,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:49,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:50,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:50,826 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:22:51,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:51,923 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:52,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:52,827 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:22:53,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:54,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:54,828 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:22:55,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:56,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:56,828 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:22:57,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:57,310 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:57,892 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:22:58,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:22:58,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:22:58,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:59,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:00,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:01,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:02,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:02,368 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:03,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:04,244 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:05,244 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:06,244 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:07,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:08,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:08,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:09,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:10,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:11,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:12,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:13,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:23:13,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:23:13,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:13,283 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:14,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:15,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:16,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:17,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:18,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:18,477 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:19,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:20,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:20,835 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:21,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:22,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:23,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:24,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:24,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:25,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:26,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:26,836 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:27,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:27,818 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:23:27,819 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 with policy now +2025-01-07 16:23:27,819 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:23:27,820 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:23:27,820 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:23:27,820 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:23:27,893 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:23:28,057 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/p9q72dzz-media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 +2025-01-07 16:23:28,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:23:28,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:23:28,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:28,314 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/iuelrx8v-media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 +2025-01-07 16:23:28,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:29,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:29,270 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:30,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:31,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:32,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:32,838 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:33,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:34,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:34,486 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:34,839 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:35,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:36,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:37,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:38,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:39,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:40,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:40,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:41,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:42,250 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:43,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:23:43,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:23:43,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:44,250 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:45,250 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:45,353 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:46,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:47,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:48,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:49,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:50,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:50,893 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:51,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:51,845 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:23:52,078 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/o4d0xdr7-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:23:52,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:52,845 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:53,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:54,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:55,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:56,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:56,847 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:56,853 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:57,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:57,893 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:23:58,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:23:58,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:23:58,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:59,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:00,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:01,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:02,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:02,343 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:03,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:04,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:04,849 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:05,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:06,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:07,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:08,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:08,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:09,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:10,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:11,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:12,254 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:13,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:24:13,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:24:13,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:13,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:14,254 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:15,254 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:16,255 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:17,255 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:18,256 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:18,854 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:19,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:19,256 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:20,256 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:21,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:22,256 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:23,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:24,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:25,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:25,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:26,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:27,258 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:27,894 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:24:28,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:24:28,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:24:28,258 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:29,258 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:30,259 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:30,739 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:30,781 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:24:30,782 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:24:30,782 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 with policy now +2025-01-07 16:24:30,782 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:24:30,783 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:24:30,793 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:24:30,867 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:30,869 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:24:30,869 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 +2025-01-07 16:24:30,870 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:24:31,013 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/xaj11f1n-media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 +2025-01-07 16:24:31,259 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:31,311 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/fltf2n5g-media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 +2025-01-07 16:24:32,263 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:32,868 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:33,263 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:34,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:34,869 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:35,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:35,861 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:36,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:37,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:38,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:38,870 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:39,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:40,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:41,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:41,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:42,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:43,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:24:43,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:24:43,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:44,265 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:45,263 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:46,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:46,312 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:47,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:47,875 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:24:47,875 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:24:47,876 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:24:47,877 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:24:48,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:48,877 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:48,877 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:24:49,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:50,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:50,878 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:51,293 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:51,410 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:52,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:52,878 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:53,265 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:54,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:55,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:56,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:57,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:57,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:57,894 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:24:58,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:24:58,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:24:58,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:59,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:00,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:01,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:02,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:02,328 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:03,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:04,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:05,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:06,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:07,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:08,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:08,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:09,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:10,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:11,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:12,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:13,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:25:13,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:25:13,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:13,305 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:14,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:15,269 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:16,269 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:17,270 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:18,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:19,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:19,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:20,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:21,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:22,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:23,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:24,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:25,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:25,272 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:26,275 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:26,893 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:27,276 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:27,530 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:25:27,530 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 with policy now +2025-01-07 16:25:27,534 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:25:27,534 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:25:27,534 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:25:27,538 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:25:27,750 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/bxthlt0r-media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 +2025-01-07 16:25:27,893 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:25:27,894 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 +2025-01-07 16:25:27,894 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:25:27,895 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:25:28,169 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/q48qsc86-media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 +2025-01-07 16:25:28,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:25:28,224 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:25:28,282 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:28,894 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:29,284 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:30,289 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:30,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:31,293 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:32,291 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:32,895 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:33,305 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:34,292 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:34,896 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:35,294 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:35,306 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:36,294 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:37,300 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:37,712 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:25:37,713 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:25:37,713 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:25:37,897 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:25:37,897 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:25:38,164 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/vt42u2z1-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:25:38,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:38,897 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:39,294 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:40,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:40,577 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:40,898 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:41,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:42,298 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:42,899 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:43,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:25:43,218 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:25:43,298 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:44,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:45,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:46,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:46,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:47,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:48,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:49,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:50,300 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:51,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:52,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:52,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:53,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:54,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:55,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:56,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:57,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:57,895 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:25:57,896 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:58,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:25:58,218 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:25:58,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:59,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:00,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:01,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:02,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:03,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:03,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:04,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:05,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:06,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:07,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:08,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:09,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:09,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:10,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:11,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:12,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:12,908 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:13,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:26:13,218 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:26:13,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:14,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:14,348 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:15,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:16,325 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:17,308 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:17,605 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:26:17,605 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:26:17,605 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 with policy now +2025-01-07 16:26:17,606 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:26:17,606 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:26:17,608 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:26:17,839 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/5d6gzbe5-media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 +2025-01-07 16:26:17,910 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:26:17,910 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 +2025-01-07 16:26:17,910 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:26:18,176 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/ydujnu7t-media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 +2025-01-07 16:26:18,307 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:18,910 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:19,310 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:20,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:20,309 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:21,309 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:22,309 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:22,912 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:23,320 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:24,317 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:24,913 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:25,319 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:25,545 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:26,311 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:27,315 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:27,895 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:26:28,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:26:28,225 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:26:28,316 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:29,315 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:30,319 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:31,322 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:31,378 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:32,323 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:33,319 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:34,319 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:35,322 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:36,321 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:37,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:37,325 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:38,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:39,328 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:40,329 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:41,332 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:41,787 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:26:41,787 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:26:41,788 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:26:41,918 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:26:41,918 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:26:42,152 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/e9249pwo-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:26:42,327 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:42,805 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:42,918 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:43,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:26:43,222 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:26:43,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:44,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:44,919 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:45,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:46,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:46,920 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:47,332 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:48,117 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:48,332 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:48,921 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:49,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:50,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:51,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:52,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:53,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:53,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:54,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:55,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:56,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:57,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:57,896 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:26:58,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:26:58,220 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:26:58,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:58,361 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:59,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:00,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:01,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:02,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:03,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:04,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:04,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:05,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:06,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:07,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:08,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:08,928 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:09,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:09,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:10,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:11,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:12,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:13,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:27:13,220 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:27:13,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:14,323 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:14,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:15,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:16,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:17,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:18,337 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:19,337 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:20,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:20,338 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:21,338 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:22,342 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:22,879 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:27:22,879 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 with policy now +2025-01-07 16:27:22,893 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:27:22,893 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:22,893 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:22,894 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:22,933 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:22,933 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:22,933 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 +2025-01-07 16:27:22,934 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:27:23,106 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/yn8zh6xg-media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 +2025-01-07 16:27:23,339 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:23,365 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/rutr07br-media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 +2025-01-07 16:27:24,341 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:24,934 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:25,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:25,348 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:26,349 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:27,350 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:27,896 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:27:28,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:27:28,227 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:27:28,355 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:28,935 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:29,351 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:30,269 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:30,349 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:30,936 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:31,349 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:32,347 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:32,850 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:32,850 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:32,850 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:32,936 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:32,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:27:33,352 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:33,568 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:27:33,568 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:27:33,715 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:27:33,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:27:33,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media +2025-01-07 16:27:33,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval +2025-01-07 16:27:33,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table +2025-01-07 16:27:34,052 INFO wandb-upload_0:2183860 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwq4f5mmo +2025-01-07 16:27:34,089 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:27:34,100 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:27:34,149 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/v79rk612-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:27:34,347 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:34,395 INFO SenderThread:2183860 [sender.py:send_request_log_artifact():1525] logged artifact run-kd9mh2vh-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc1NzM1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY1MjM5Mw==', 'latestArtifact': None}} +2025-01-07 16:27:34,395 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:27:34,395 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-07 16:27:34,395 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:34,396 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,396 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,396 DEBUG SenderThread:2183860 [sender.py:send():379] send: telemetry +2025-01-07 16:27:34,396 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,400 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,401 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,401 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,401 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,401 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,402 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,402 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,402 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,402 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,402 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,403 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,403 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,403 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,403 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,404 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,404 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,404 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,404 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,405 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,405 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,405 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,405 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:27:34,633 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/fs8tvijo-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:27:34,787 INFO wandb-upload_1:2183860 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd6gijgim +2025-01-07 16:27:34,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:34,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:34,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:27:34,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json +2025-01-07 16:27:34,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval +2025-01-07 16:27:35,176 INFO SenderThread:2183860 [sender.py:send_request_log_artifact():1525] logged artifact run-kd9mh2vh-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc1NzM5NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY1MjM5OA==', 'latestArtifact': None}} +2025-01-07 16:27:35,176 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:27:35,176 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json with policy now +2025-01-07 16:27:35,176 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:35,177 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:35,177 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:35,347 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:35,353 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/0c5b3dxk-media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json +2025-01-07 16:27:35,594 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/14edwdcm-media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json +2025-01-07 16:27:35,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:36,141 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:36,244 DEBUG SenderThread:2183860 [sender.py:send():379] send: exit +2025-01-07 16:27:36,244 INFO SenderThread:2183860 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 16:27:36,244 INFO SenderThread:2183860 [sender.py:send_exit():588] handling runtime: 488 +2025-01-07 16:27:36,244 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:36,244 INFO SenderThread:2183860 [sender.py:send_exit():594] send defer +2025-01-07 16:27:36,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,245 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 16:27:36,245 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:36,245 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 16:27:36,245 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 1 +2025-01-07 16:27:36,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,245 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 16:27:36,245 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:36,245 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 16:27:36,245 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 2 +2025-01-07 16:27:36,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,245 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 16:27:36,246 INFO HandlerThread:2183860 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 16:27:36,246 DEBUG SystemMonitor:2183860 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 16:27:36,246 DEBUG SystemMonitor:2183860 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 16:27:36,246 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 16:27:36,247 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 16:27:36,298 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 16:27:36,298 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 16:27:36,298 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined network monitor +2025-01-07 16:27:36,299 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:36,299 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 16:27:36,299 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 3 +2025-01-07 16:27:36,299 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:27:36,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,299 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 16:27:36,299 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:36,299 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 16:27:36,299 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 4 +2025-01-07 16:27:36,300 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,300 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 16:27:36,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:36,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:36,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml +2025-01-07 16:27:42,856 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:42,856 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:42,856 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 16:27:42,857 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:42,857 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 16:27:42,857 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 5 +2025-01-07 16:27:42,857 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:42,857 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:42,857 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:42,858 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:42,858 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:42,858 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 16:27:42,859 DEBUG SenderThread:2183860 [sender.py:send():379] send: summary +2025-01-07 16:27:42,859 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:42,859 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:42,859 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 16:27:42,859 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 6 +2025-01-07 16:27:42,859 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:42,859 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 16:27:42,859 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:42,859 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 16:27:42,860 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 7 +2025-01-07 16:27:42,860 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:42,860 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:42,860 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 16:27:42,860 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:42,860 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 16:27:42,940 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:43,183 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:44,931 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 8 +2025-01-07 16:27:44,931 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:44,931 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:44,931 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 16:27:44,931 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:44,931 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 16:27:44,931 INFO SenderThread:2183860 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 16:27:44,934 INFO SenderThread:2183860 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 16:27:44,936 INFO SenderThread:2183860 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 16:27:44,941 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:44,946 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 9 +2025-01-07 16:27:44,946 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:44,946 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 16:27:44,947 DEBUG SenderThread:2183860 [sender.py:send():379] send: artifact +2025-01-07 16:27:45,183 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:45,606 INFO wandb-upload_0:2183860 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp5_jfea2 +2025-01-07 16:27:45,816 INFO wandb-upload_1:2183860 [upload_job.py:push():88] Uploaded file /tmp/tmpe_ev39sb/wandb-job.json +2025-01-07 16:27:46,356 INFO SenderThread:2183860 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjY1NzQxMQ==', 'versionIndex': 430}}} +2025-01-07 16:27:46,356 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:46,356 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 16:27:46,356 INFO SenderThread:2183860 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 16:27:46,942 INFO SenderThread:2183860 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files +2025-01-07 16:27:46,942 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml config.yaml +2025-01-07 16:27:46,942 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json wandb-summary.json +2025-01-07 16:27:46,943 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/requirements.txt requirements.txt +2025-01-07 16:27:46,943 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:27:46,943 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-metadata.json wandb-metadata.json +2025-01-07 16:27:46,944 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log output.log +2025-01-07 16:27:46,948 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 16:27:46,948 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 +2025-01-07 16:27:46,949 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 +2025-01-07 16:27:46,949 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 +2025-01-07 16:27:46,950 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 +2025-01-07 16:27:46,950 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 +2025-01-07 16:27:46,952 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json +2025-01-07 16:27:46,952 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:27:46,952 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 10 +2025-01-07 16:27:46,952 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:46,952 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:46,952 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 16:27:46,953 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:46,953 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 16:27:46,953 INFO SenderThread:2183860 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:27:47,119 INFO wandb-upload_2:2183860 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:47,144 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml +2025-01-07 16:27:47,170 INFO wandb-upload_4:2183860 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:47,184 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:47,185 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:47,198 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/requirements.txt +2025-01-07 16:27:47,221 INFO wandb-upload_3:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/2t6nakx0-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:27:47,422 INFO Thread-11 (_thread_body):2183860 [sender.py:transition_state():614] send defer: 11 +2025-01-07 16:27:47,422 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:47,422 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 16:27:47,422 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:47,422 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 16:27:47,422 INFO SenderThread:2183860 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 16:27:47,424 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 12 +2025-01-07 16:27:47,424 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:47,424 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 16:27:47,424 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:47,424 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 16:27:47,424 INFO SenderThread:2183860 [file_stream.py:finish():601] file stream finish called +2025-01-07 16:27:47,600 INFO SenderThread:2183860 [file_stream.py:finish():605] file stream finish is done +2025-01-07 16:27:47,600 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 13 +2025-01-07 16:27:47,600 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:47,601 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 16:27:47,601 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:47,601 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 16:27:47,601 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 14 +2025-01-07 16:27:47,601 DEBUG SenderThread:2183860 [sender.py:send():379] send: final +2025-01-07 16:27:47,601 DEBUG SenderThread:2183860 [sender.py:send():379] send: footer +2025-01-07 16:27:47,601 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:47,601 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 16:27:47,602 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:47,602 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 16:27:47,602 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:47,602 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 16:27:47,602 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:47,603 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 16:27:47,603 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 16:27:47,604 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:47,604 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: server_info +2025-01-07 16:27:47,604 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:47,655 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:47,655 INFO MainThread:2183860 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 16:27:47,656 INFO MainThread:2183860 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 16:27:47,656 INFO MainThread:2183860 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 16:27:47,657 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 16:27:47,657 INFO HandlerThread:2183860 [handler.py:finish():882] shutting down handler +2025-01-07 16:27:48,603 INFO WriterThread:2183860 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/run-kd9mh2vh.wandb +2025-01-07 16:27:48,655 INFO SenderThread:2183860 [sender.py:finish():1615] shutting down sender +2025-01-07 16:27:48,655 INFO SenderThread:2183860 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:27:48,655 INFO SenderThread:2183860 [file_pusher.py:join():175] waiting for file pusher diff --git a/kd9mh2vh/wandb/debug.log b/kd9mh2vh/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..454dfa4b768ad247770cb7dc5d5026ca04c9987a --- /dev/null +++ b/kd9mh2vh/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Configure stats pid to 2183494 +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/settings +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug.log +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug-internal.log +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_init.py:init():569] calling init triggers +2025-01-07 16:19:27,461 INFO MainThread:2183494 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh'} +2025-01-07 16:19:27,461 INFO MainThread:2183494 [wandb_init.py:init():619] starting backend +2025-01-07 16:19:27,461 INFO MainThread:2183494 [wandb_init.py:init():623] setting up manager +2025-01-07 16:19:27,465 INFO MainThread:2183494 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 16:19:27,466 INFO MainThread:2183494 [wandb_init.py:init():631] backend started and connected +2025-01-07 16:19:27,479 INFO MainThread:2183494 [wandb_init.py:init():720] updated telemetry +2025-01-07 16:19:27,487 INFO MainThread:2183494 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 16:19:27,786 INFO MainThread:2183494 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 16:19:27,857 INFO MainThread:2183494 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 16:19:27,857 INFO MainThread:2183494 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 16:19:28,214 INFO MainThread:2183494 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 16:19:28,214 INFO MainThread:2183494 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 16:19:28,214 INFO MainThread:2183494 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 16:19:28,214 INFO MainThread:2183494 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 16:19:28,215 INFO MainThread:2183494 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 16:19:28,216 INFO MainThread:2183494 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/tb, True +2025-01-07 16:19:53,104 INFO MainThread:2183494 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 16:27:48,658 WARNING MsgRouterThr:2183494 [router.py:message_loop():77] message_loop has been closed diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code/FISH/eval_robot.py b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66f0385425be8da946d1ca7a42da1a971f658125 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736284767 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..ba7769f66c979b1d232be907f687af5981f84c15 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[27, 35, 0.7714285714285715, 0.5759195685234819, 0.9056934117459488, 0.6182901412012032, 0.894987322311082, 0.644098646909981, 0.8680112733700966, 0.6607312969541952, 0.8597671537871852]]} \ No newline at end of file diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57bdafb889d1c369058938333621155f869caeb4 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89262cab025e888a6f21af0c8e0e45f248aec311e89fe6eff8c0a6420dd9edd7 +size 1537570 diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c04447e082e0c64c4187aa6938a4350de9ad776e --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:887535ba826c760b568aab1c6a63fa73d25e9aedf01bed19177ee99551a8a746 +size 1531638 diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c9a87e79e1d951c8acbb11ce27e64fcbaaacfeb1 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de6316f5b26b7e9850a50c030889b874192ec1e76b280f906cc47e75c46cbbfb +size 1527940 diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2abaa7737badc4afe11282c13b07a7e77e110b4b --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:295a37e1d2a179eff24306db755a7658dce0634a5554cc23e6d829e32b922edc +size 1544801 diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..275cdd693d55cdfe56d93f61ee7a3e0c90b6f148 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a7a2360b5bcbe4011d794f92c18f5dea65247cc82a1a66fedb61fcdde9575b +size 1548137 diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4a4304232cb01433ba3b4c17601df062753098ca --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736284793.105810]: resetting environment +[INFO] [1736284793.111902]: cleared current plan +[INFO] [1736284793.112960]: moving to home +[INFO] [1736284795.616186]: reached home +[INFO] [1736284795.617602]: reset action history +[INFO] [1736284798.330801]: environment reset +Starting episode 0 +[INFO] [1736284798.335134]: resetting environment +[INFO] [1736284798.341120]: cleared current plan +[INFO] [1736284798.342008]: moving to home +[INFO] [1736284799.344196]: reached home +[INFO] [1736284799.345541]: reset action history +[INFO] [1736284802.055479]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736284968.206283]: resetting environment +[INFO] [1736284968.211147]: cleared current plan +[INFO] [1736284968.211949]: moving to home +[INFO] [1736284969.214089]: reached home +[INFO] [1736284969.215685]: reset action history +[INFO] [1736284971.922788]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/episode_rosbags/episode_0_2025-01-07-16-22-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2183494.log +started roslaunch server http://158.130.50.37:35641/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2183973] +process[print_text-2]: started with pid [2183974] +process[rosbag_record-3]: started with pid [2183997] +started bagging! +For topic gripper_width: timestamp difference is -102932599 for nearest: 1736284974078873488 - target: 1736284974181806087 at idx 27 +For topic gripper_width: timestamp difference is -104921572 for nearest: 1736284998278900392 - target: 1736284998383821964 at idx 249 +For topic gripper_width: timestamp difference is 100101511 for nearest: 1736285004745539228 - target: 1736285004645437717 at idx 249 +[INFO] [1736285006.806139]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736285007.576825]: Stored episode 1. +[INFO] [1736285007.577074]: Saving video... +[INFO] [1736285007.802879]: Video saved! +Starting episode 1 +[INFO] [1736285007.819877]: resetting environment +[INFO] [1736285007.831033]: cleared current plan +[INFO] [1736285007.831765]: moving to home +[INFO] [1736285010.734793]: reached home +[INFO] [1736285010.735248]: reset action history +[INFO] [1736285013.444247]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736285030.896855]: resetting environment +[INFO] [1736285030.901368]: cleared current plan +[INFO] [1736285030.901540]: moving to home +[INFO] [1736285031.902359]: reached home +[INFO] [1736285031.902765]: reset action history +[INFO] [1736285034.614832]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/episode_rosbags/episode_1_2025-01-07-16-23-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2183494.log +started roslaunch server http://158.130.50.37:39871/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2184123] +process[print_text-5]: started with pid [2184124] +process[rosbag_record-6]: started with pid [2184125] +started bagging! +For topic gripper_width: timestamp difference is 100131403 for nearest: 1736285042845539938 - target: 1736285042745408535 at idx 104 +For topic gripper_width: timestamp difference is 109543455 for nearest: 1736285057645564449 - target: 1736285057536020994 at idx 249 +[INFO] [1736285069.399783]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736285070.159579]: Stored episode 2. +[INFO] [1736285070.160007]: Saving video... +[INFO] [1736285070.742891]: Video saved! +Starting episode 2 +[INFO] [1736285070.779021]: resetting environment +[INFO] [1736285070.809499]: cleared current plan +[INFO] [1736285070.809942]: moving to home +[WARN] [1736285070.731064]: Plan exhausted +[WARN] [1736285070.770686]: Plan exhausted +[INFO] [1736285073.813398]: reached home +[INFO] [1736285073.814172]: reset action history +[INFO] [1736285076.540075]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736285087.526441]: resetting environment +[INFO] [1736285087.541108]: cleared current plan +[INFO] [1736285087.541667]: moving to home +[INFO] [1736285088.543140]: reached home +[INFO] [1736285088.543645]: reset action history +[INFO] [1736285091.251443]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/episode_rosbags/episode_2_2025-01-07-16-24-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2183494.log +started roslaunch server http://158.130.50.37:45029/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2184268] +process[print_text-8]: started with pid [2184269] +process[rosbag_record-9]: started with pid [2184293] +started bagging! +[INFO] [1736285126.031671]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736285126.788874]: Stored episode 3. +[INFO] [1736285126.797835]: Saving video... +[INFO] [1736285127.495393]: Video saved! +Starting episode 3 +[INFO] [1736285127.539325]: resetting environment +[INFO] [1736285127.591663]: cleared current plan +[INFO] [1736285127.609428]: moving to home +[WARN] [1736285127.364617]: Plan exhausted +[WARN] [1736285127.392607]: Plan exhausted +[WARN] [1736285127.435817]: Plan exhausted +[WARN] [1736285127.472557]: Plan exhausted +[WARN] [1736285127.516353]: Plan exhausted +[WARN] [1736285127.556538]: Plan exhausted +[INFO] [1736285130.522918]: reached home +[INFO] [1736285130.524014]: reset action history +[INFO] [1736285133.261131]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736285137.523797]: resetting environment +[INFO] [1736285137.532941]: cleared current plan +[INFO] [1736285137.533544]: moving to home +[INFO] [1736285138.535260]: reached home +[INFO] [1736285138.539013]: reset action history +[INFO] [1736285141.259989]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/episode_rosbags/episode_3_2025-01-07-16-25-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2183494.log +started roslaunch server http://158.130.50.37:33853/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2184432] +process[print_text-11]: started with pid [2184433] +process[rosbag_record-12]: started with pid [2184434] +started bagging! +For topic gripper_width: timestamp difference is 101698004 for nearest: 1736285171878871523 - target: 1736285171777173519 at idx 249 +[INFO] [1736285176.283136]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736285177.069142]: Stored episode 4. +[INFO] [1736285177.077420]: Saving video... +[INFO] [1736285177.558449]: Video saved! +Starting episode 4 +[INFO] [1736285177.613265]: resetting environment +[INFO] [1736285177.649876]: cleared current plan +[INFO] [1736285177.651271]: moving to home +[WARN] [1736285177.631735]: Plan exhausted +[INFO] [1736285180.756833]: reached home +[INFO] [1736285180.761738]: reset action history +[INFO] [1736285183.496150]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736285201.770193]: resetting environment +[INFO] [1736285201.801028]: cleared current plan +[INFO] [1736285201.803090]: moving to home +[INFO] [1736285202.813344]: reached home +[INFO] [1736285202.817173]: reset action history +[INFO] [1736285205.544336]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/episode_rosbags/episode_4_2025-01-07-16-26-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2183494.log +started roslaunch server http://158.130.50.37:39927/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2184574] +process[print_text-14]: started with pid [2184575] +process[rosbag_record-15]: started with pid [2184576] +started bagging! +For topic gripper_width: timestamp difference is 117812551 for nearest: 1736285227545565523 - target: 1736285227427752972 at idx 249 +[INFO] [1736285241.626038]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit[print_text-14] killing on exit +[INFO] [1736285242.425013]: Stored episode 5. +[INFO] [1736285242.435871]: Saving video... +[INFO] [1736285242.822407]: Video saved! +[INFO] [1736285242.894629]: resetting environment +[INFO] [1736285242.921130]: cleared current plan +[INFO] [1736285242.921751]: moving to home +[INFO] [1736285246.426506]: reached home +[INFO] [1736285246.433331]: reset action history +[INFO] [1736285249.227974]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/requirements.txt b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-metadata.json b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3560afe6c5d9ebc1f5fafcebb210cd67ef0e8521 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T21:19:27.913455", + "startedAt": "2025-01-07T21:19:27.458529", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.7955, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.992, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.029, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4502.343, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 650.7937316894531 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d4b0e296b2dcc26b85f89bbb0c355ad33a626e24 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "89262cab025e888a6f21af0c8e0e45f248aec311e89fe6eff8c0a6420dd9edd7", "size": 1537570, "path": "media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4"}, "global_step": 665, "_timestamp": 1736285254.0120993, "_runtime": 486.54433131217957, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "887535ba826c760b568aab1c6a63fa73d25e9aedf01bed19177ee99551a8a746", "size": 1531638, "path": "media/videos/eval/1_eval_1_887535ba826c760b568a.mp4"}, "eval/num_success": 1.0, "episode": 35.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "de6316f5b26b7e9850a50c030889b874192ec1e76b280f906cc47e75c46cbbfb", "size": 1527940, "path": "media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "295a37e1d2a179eff24306db755a7658dce0634a5554cc23e6d829e32b922edc", "size": 1544801, "path": "media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "a8a7a2360b5bcbe4011d794f92c18f5dea65247cc82a1a66fedb61fcdde9575b", "size": 1548137, "path": "media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://4t5isff1wzu0l2jr0kby9jl7xa6oj2jwuma95a0kvbugvxyyqi3vuycocwteoonulctlla4sbz5qvlgbkog0hxu4pwnk2om0j21bxvs9pakhftcyw7e4zxlgkjfej2n1/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://p2ca7r0on808wnwmeos971ogmxcpiwm9g3vqv0gjkqn397tzh6ym79icqqpy8o1uj33drct96t8tngrnsokvznv9ircb1jcmeza8w7ozcxvp3xnbcfqj9megi0cgtwbu:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 27, "total_num_episodes": 35, "total_success_rate": 0.7714285714285715, "total_success_rate_lb_0.01": 0.5759195685234819, "total_success_rate_ub_0.01": 0.9056934117459488, "total_success_rate_lb_0.025": 0.6182901412012032, "total_success_rate_ub_0.025": 0.894987322311082, "total_success_rate_lb_0.05": 0.644098646909981, "total_success_rate_ub_0.05": 0.8680112733700966, "total_success_rate_lb_0.1": 0.6607312969541952, "total_success_rate_ub_0.1": 0.8597671537871852, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "6360f159e4e74e6b412e1d490c66c2321499c61e8c91bb7ee142dc6920326616", "size": 518, "artifact_path": "wandb-client-artifact://cdcddqqmfzgua2pqf3s7hnn36dtxxoj70tdle7hf97hb7annt6jacom5sfl5xr7ywo2x4abo3mq5kdpi5zcl7u9x0eveyr79hcvbsjnfmnlazzwh7wgua08typfr53of/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://voumb7zt8yuvlp8qidi7d5ufa6jmx5d8z0scruzd5ctimaamgwsa9vvzh35f4kszw3eb4f40gi1he6r2nbxpp2hcf8a4om1uxdpenh1rfxy6lupz9mxhb8qqv2sola4b:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 488}, "eval/total_success_rate": 0.7714285850524902, "eval/total_success_rate_lb_0.01": 0.5759195685386658, "eval/total_success_rate_ub_0.01": 0.9056934118270874, "eval/total_success_rate_lb_0.025": 0.6182901263237, "eval/total_success_rate_ub_0.025": 0.8949873447418213, "eval/total_success_rate_lb_0.05": 0.6440986394882202, "eval/total_success_rate_ub_0.05": 0.8680112957954407, "eval/total_success_rate_lb_0.1": 0.660731315612793, "eval/total_success_rate_ub_0.1": 0.8597671389579773} \ No newline at end of file diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug-internal.log b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a094fc542e475b6ef1312cf05b7773ff7a02ab08 --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug-internal.log @@ -0,0 +1,1069 @@ +2025-01-07 16:19:27,467 INFO StreamThr :2183860 [internal.py:wandb_internal():85] W&B internal server running at pid: 2183860, started at: 2025-01-07 16:19:27.466675 +2025-01-07 16:19:27,468 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status +2025-01-07 16:19:27,479 INFO WriterThread:2183860 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/run-kd9mh2vh.wandb +2025-01-07 16:19:27,480 DEBUG SenderThread:2183860 [sender.py:send():379] send: header +2025-01-07 16:19:27,490 DEBUG SenderThread:2183860 [sender.py:send():379] send: run +2025-01-07 16:19:27,771 INFO SenderThread:2183860 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files +2025-01-07 16:19:27,771 INFO SenderThread:2183860 [sender.py:_start_run_threads():1188] run started: kd9mh2vh with start time 1736284767.467768 +2025-01-07 16:19:27,787 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 16:19:27,787 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: check_version +2025-01-07 16:19:27,862 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 16:19:27,885 DEBUG HandlerThread:2183860 [system_info.py:__init__():26] System info init +2025-01-07 16:19:27,885 DEBUG HandlerThread:2183860 [system_info.py:__init__():41] System info init done +2025-01-07 16:19:27,885 INFO HandlerThread:2183860 [system_monitor.py:start():194] Starting system monitor +2025-01-07 16:19:27,885 INFO SystemMonitor:2183860 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 16:19:27,886 INFO HandlerThread:2183860 [system_monitor.py:probe():214] Collecting system info +2025-01-07 16:19:27,886 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 16:19:27,887 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started disk monitoring +2025-01-07 16:19:27,887 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 16:19:27,888 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started memory monitoring +2025-01-07 16:19:27,888 INFO SystemMonitor:2183860 [interfaces.py:start():188] Started network monitoring +2025-01-07 16:19:27,913 DEBUG HandlerThread:2183860 [system_info.py:probe():152] Probing system +2025-01-07 16:19:27,914 DEBUG HandlerThread:2183860 [system_info.py:_probe_git():137] Probing git +2025-01-07 16:19:27,920 DEBUG HandlerThread:2183860 [system_info.py:_probe_git():145] Probing git done +2025-01-07 16:19:27,920 DEBUG HandlerThread:2183860 [system_info.py:probe():200] Probing system done +2025-01-07 16:19:27,920 DEBUG HandlerThread:2183860 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:19:27.913455', 'startedAt': '2025-01-07T21:19:27.458529', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.7955, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.992, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.029, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4502.343, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 650.7937316894531}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 16:19:27,920 INFO HandlerThread:2183860 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 16:19:27,920 INFO HandlerThread:2183860 [system_monitor.py:probe():227] Publishing system info +2025-01-07 16:19:27,920 DEBUG HandlerThread:2183860 [system_info.py:_save_code():44] Saving code +2025-01-07 16:19:27,922 DEBUG HandlerThread:2183860 [system_info.py:_save_code():67] Saving code done +2025-01-07 16:19:27,922 DEBUG HandlerThread:2183860 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 16:19:27,952 DEBUG HandlerThread:2183860 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 16:19:27,952 INFO HandlerThread:2183860 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 16:19:27,955 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:19:27,955 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 16:19:27,955 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 16:19:28,202 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/a5pvksfa-wandb-metadata.json +2025-01-07 16:19:28,208 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/p21bpd4i-code/FISH/eval_robot.py +2025-01-07 16:19:28,213 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 16:19:28,214 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: python_packages +2025-01-07 16:19:28,214 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:19:28,214 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:19:28,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:28,216 INFO HandlerThread:2183860 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/tb" + save: true +} +control { + relay_id: "fe15f45b5ac547a1abc47cd88a1a163b" +} +_info { + stream_id: "kd9mh2vh" +} + +2025-01-07 16:19:28,216 DEBUG HandlerThread:2183860 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 16:19:28,311 DEBUG SenderThread:2183860 [sender.py:send():379] send: telemetry +2025-01-07 16:19:28,311 DEBUG SenderThread:2183860 [sender.py:send():379] send: tbrecord +2025-01-07 16:19:28,311 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:19:28,311 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 with policy live +2025-01-07 16:19:28,485 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/wmziujzy-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:19:28,773 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:19:28,773 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code/FISH/eval_robot.py +2025-01-07 16:19:28,773 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-metadata.json +2025-01-07 16:19:28,773 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/requirements.txt +2025-01-07 16:19:28,774 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:19:28,774 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code/FISH +2025-01-07 16:19:28,774 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code +2025-01-07 16:19:29,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:30,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:30,772 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:19:31,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:32,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:33,214 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:33,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:34,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:35,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:36,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:37,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:38,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:38,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:39,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:40,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:41,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:42,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:43,214 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:19:43,214 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:19:43,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:43,281 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:44,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:45,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:46,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:47,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:48,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:49,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:49,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:50,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:51,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:52,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:53,105 DEBUG SenderThread:2183860 [sender.py:send():379] send: config +2025-01-07 16:19:53,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:54,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:54,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:54,779 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:19:55,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:56,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:56,779 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:19:57,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:58,214 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:19:58,214 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:19:58,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:59,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:19:59,348 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:19:59,780 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml +2025-01-07 16:20:00,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:00,780 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:20:01,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:02,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:03,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:04,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:04,781 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:20:05,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:05,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:06,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:07,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:08,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:09,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:10,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:10,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:11,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:12,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:13,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:20:13,215 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:20:13,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:14,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:15,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:15,314 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:16,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:17,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:18,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:19,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:20,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:21,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:21,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:22,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:23,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:24,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:25,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:26,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:26,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:27,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:27,889 DEBUG SystemMonitor:2183860 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 16:20:27,889 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:20:28,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:20:28,215 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:20:28,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:29,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:30,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:31,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:31,380 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:32,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:33,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:34,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:35,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:36,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:37,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:37,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:38,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:39,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:40,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:41,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:42,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:43,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:20:43,215 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:20:43,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:43,310 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:44,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:45,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:46,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:47,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:48,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:49,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:49,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:50,226 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:51,226 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:52,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:53,226 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:54,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:20:54,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:55,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:56,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:57,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:57,890 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:20:58,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:20:58,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:20:58,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:59,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:20:59,308 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:00,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:01,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:02,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:03,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:04,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:05,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:05,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:06,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:07,228 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:08,229 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:09,229 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:10,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:10,229 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:11,229 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:12,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:13,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:21:13,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:21:13,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:14,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:15,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:15,366 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:16,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:17,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:18,230 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:19,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:20,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:21,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:21,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:22,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:23,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:24,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:25,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:26,218 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:26,231 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:27,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:27,890 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:21:28,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:21:28,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:21:28,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:29,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:30,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:31,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:31,291 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:32,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:33,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:34,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:35,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:36,232 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:37,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:37,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:38,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:39,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:40,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:41,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:42,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:42,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:43,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:21:43,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:21:43,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:44,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:45,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:46,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:47,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:47,289 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:48,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:49,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:50,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:51,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:52,234 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:53,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:53,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:54,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:55,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:56,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:57,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:57,891 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:21:58,215 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:21:58,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:21:58,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:21:58,321 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:21:59,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:00,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:01,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:02,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:03,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:04,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:04,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:05,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:06,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:07,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:08,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:09,219 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:09,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:10,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:11,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:12,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:13,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:22:13,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:22:13,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:14,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:14,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:15,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:16,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:17,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:18,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:19,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:20,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:20,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:21,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:22,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:23,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:24,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:25,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:25,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:26,238 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:27,239 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:27,891 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:22:28,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:22:28,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:22:28,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:29,239 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:30,239 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:30,295 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:31,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:32,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:33,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:34,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:35,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:36,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:36,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:37,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:38,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:39,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:40,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:41,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:41,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:42,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:43,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:22:43,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:22:43,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:44,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:45,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:46,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:46,360 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:47,241 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:48,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:49,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:50,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:50,826 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:22:51,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:51,923 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:52,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:52,827 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:22:53,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:54,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:54,828 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:22:55,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:56,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:56,828 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:22:57,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:57,310 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:22:57,892 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:22:58,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:22:58,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:22:58,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:22:59,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:00,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:01,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:02,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:02,368 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:03,243 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:04,244 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:05,244 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:06,244 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:07,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:08,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:08,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:09,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:10,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:11,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:12,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:13,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:23:13,216 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:23:13,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:13,283 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:14,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:15,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:16,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:17,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:18,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:18,477 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:19,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:20,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:20,835 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:21,246 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:22,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:23,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:24,221 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:24,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:25,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:26,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:26,836 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:27,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:27,818 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:23:27,819 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 with policy now +2025-01-07 16:23:27,819 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:23:27,820 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:23:27,820 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:23:27,820 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos +2025-01-07 16:23:27,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:23:27,893 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:23:28,057 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/p9q72dzz-media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 +2025-01-07 16:23:28,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:23:28,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:23:28,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:28,314 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/iuelrx8v-media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 +2025-01-07 16:23:28,837 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:29,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:29,270 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:30,247 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:31,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:32,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:32,838 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:33,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:34,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:34,486 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:34,839 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:35,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:36,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:37,248 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:38,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:39,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:40,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:40,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:41,249 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:42,250 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:43,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:23:43,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:23:43,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:44,250 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:45,250 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:45,353 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:46,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:47,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:48,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:49,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:50,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:50,893 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:51,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:51,845 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:23:52,078 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/o4d0xdr7-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:23:52,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:52,845 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:53,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:54,251 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:55,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:56,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:56,847 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:23:56,853 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:23:57,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:57,893 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:23:58,216 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:23:58,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:23:58,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:23:59,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:00,252 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:01,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:02,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:02,343 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:03,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:04,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:04,849 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:05,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:06,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:07,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:08,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:08,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:09,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:10,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:11,253 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:12,254 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:13,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:24:13,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:24:13,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:13,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:14,254 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:15,254 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:16,255 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:17,255 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:18,256 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:18,854 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:19,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:19,256 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:20,256 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:21,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:22,256 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:23,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:24,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:25,223 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:25,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:26,257 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:27,258 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:27,894 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:24:28,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:24:28,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:24:28,258 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:29,258 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:30,259 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:30,739 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:30,781 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:24:30,782 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:24:30,782 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 with policy now +2025-01-07 16:24:30,782 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:24:30,783 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:24:30,793 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:24:30,867 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:30,869 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:24:30,869 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 +2025-01-07 16:24:30,870 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:24:31,013 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/xaj11f1n-media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 +2025-01-07 16:24:31,259 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:31,311 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/fltf2n5g-media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 +2025-01-07 16:24:32,263 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:32,868 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:33,263 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:34,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:34,869 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:35,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:35,861 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:36,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:37,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:38,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:38,870 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:39,261 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:40,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:41,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:41,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:42,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:43,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:24:43,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:24:43,262 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:44,265 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:45,263 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:46,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:46,312 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:47,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:47,875 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:24:47,875 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:24:47,876 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:24:47,877 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:24:48,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:48,877 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:48,877 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:24:49,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:50,264 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:50,878 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:51,293 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:51,410 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:52,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:52,878 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:24:53,265 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:54,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:55,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:56,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:57,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:24:57,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:57,894 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:24:58,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:24:58,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:24:58,266 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:24:59,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:00,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:01,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:02,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:02,328 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:03,267 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:04,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:05,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:06,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:07,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:08,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:08,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:09,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:10,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:11,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:12,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:13,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:25:13,217 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:25:13,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:13,305 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:14,268 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:15,269 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:16,269 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:17,270 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:18,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:19,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:19,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:20,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:21,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:22,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:23,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:24,271 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:25,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:25,272 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:26,275 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:26,893 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:27,276 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:27,530 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:25:27,530 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 with policy now +2025-01-07 16:25:27,534 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:25:27,534 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:25:27,534 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:25:27,538 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:25:27,750 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/bxthlt0r-media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 +2025-01-07 16:25:27,893 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:25:27,894 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 +2025-01-07 16:25:27,894 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:25:27,895 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:25:28,169 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/q48qsc86-media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 +2025-01-07 16:25:28,224 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:25:28,224 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:25:28,282 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:28,894 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:29,284 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:30,289 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:30,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:31,293 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:32,291 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:32,895 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:33,305 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:34,292 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:34,896 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:35,294 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:35,306 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:36,294 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:37,300 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:37,712 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:25:37,713 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:25:37,713 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:25:37,897 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:25:37,897 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:25:38,164 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/vt42u2z1-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:25:38,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:38,897 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:39,294 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:40,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:40,577 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:40,898 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:41,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:42,298 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:42,899 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:25:43,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:25:43,218 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:25:43,298 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:44,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:45,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:46,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:46,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:47,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:48,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:49,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:50,300 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:51,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:52,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:52,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:53,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:54,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:55,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:56,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:57,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:57,895 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:25:57,896 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:25:58,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:25:58,218 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:25:58,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:25:59,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:00,301 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:01,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:02,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:03,235 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:03,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:04,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:05,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:06,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:07,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:08,302 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:09,233 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:09,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:10,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:11,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:12,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:12,908 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:13,217 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:26:13,218 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:26:13,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:14,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:14,348 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:15,303 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:16,325 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:17,308 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:17,605 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:26:17,605 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:26:17,605 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 with policy now +2025-01-07 16:26:17,606 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:26:17,606 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:26:17,608 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:26:17,839 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/5d6gzbe5-media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 +2025-01-07 16:26:17,910 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:26:17,910 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 +2025-01-07 16:26:17,910 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:26:18,176 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/ydujnu7t-media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 +2025-01-07 16:26:18,307 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:18,910 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:19,310 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:20,242 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:20,309 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:21,309 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:22,309 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:22,912 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:23,320 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:24,317 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:24,913 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:25,319 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:25,545 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:26,311 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:27,315 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:27,895 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:26:28,225 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:26:28,225 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:26:28,316 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:29,315 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:30,319 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:31,322 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:31,378 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:32,323 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:33,319 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:34,319 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:35,322 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:36,321 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:37,240 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:37,325 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:38,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:39,328 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:40,329 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:41,332 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:41,787 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:26:41,787 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:26:41,788 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:26:41,918 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:26:41,918 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:26:42,152 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/e9249pwo-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:26:42,327 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:42,805 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:42,918 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:43,222 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:26:43,222 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:26:43,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:44,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:44,919 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:45,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:46,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:46,920 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:47,332 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:48,117 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:48,332 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:48,921 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:26:49,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:50,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:51,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:52,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:53,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:53,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:54,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:55,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:56,333 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:57,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:57,896 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:26:58,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:26:58,220 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:26:58,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:26:58,361 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:26:59,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:00,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:01,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:02,334 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:03,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:04,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:04,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:05,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:06,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:07,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:08,335 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:08,928 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:09,236 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:09,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:10,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:11,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:12,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:13,220 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:27:13,220 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:27:13,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:14,323 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:14,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:15,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:16,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:17,336 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:18,337 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:19,337 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:20,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:20,338 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:21,338 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:22,342 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:22,879 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:27:22,879 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 with policy now +2025-01-07 16:27:22,893 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:27:22,893 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:22,893 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:22,894 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:22,933 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:22,933 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:22,933 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 +2025-01-07 16:27:22,934 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval +2025-01-07 16:27:23,106 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/yn8zh6xg-media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 +2025-01-07 16:27:23,339 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:23,365 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/rutr07br-media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 +2025-01-07 16:27:24,341 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:24,934 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:25,237 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:25,348 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:26,349 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:27,350 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:27,896 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:27:28,227 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:27:28,227 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:27:28,355 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:28,935 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:29,351 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:30,269 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:30,349 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:30,936 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:31,349 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:32,347 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:32,850 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:32,850 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:32,850 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:32,936 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:32,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:27:33,352 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:33,568 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:27:33,568 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:27:33,715 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:27:33,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:27:33,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media +2025-01-07 16:27:33,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval +2025-01-07 16:27:33,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table +2025-01-07 16:27:34,052 INFO wandb-upload_0:2183860 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwq4f5mmo +2025-01-07 16:27:34,089 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:27:34,100 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:27:34,149 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/v79rk612-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:27:34,347 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:34,395 INFO SenderThread:2183860 [sender.py:send_request_log_artifact():1525] logged artifact run-kd9mh2vh-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc1NzM1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY1MjM5Mw==', 'latestArtifact': None}} +2025-01-07 16:27:34,395 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:27:34,395 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-07 16:27:34,395 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:34,396 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,396 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,396 DEBUG SenderThread:2183860 [sender.py:send():379] send: telemetry +2025-01-07 16:27:34,396 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,400 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,401 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,401 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,401 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,401 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,402 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,402 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,402 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,402 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,402 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,403 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,403 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,403 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,403 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,404 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,404 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,404 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,404 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,405 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,405 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:34,405 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:34,405 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:27:34,633 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/fs8tvijo-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:27:34,787 INFO wandb-upload_1:2183860 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd6gijgim +2025-01-07 16:27:34,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:34,937 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:34,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:27:34,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json +2025-01-07 16:27:34,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval +2025-01-07 16:27:35,176 INFO SenderThread:2183860 [sender.py:send_request_log_artifact():1525] logged artifact run-kd9mh2vh-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc1NzM5NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY1MjM5OA==', 'latestArtifact': None}} +2025-01-07 16:27:35,176 DEBUG SenderThread:2183860 [sender.py:send():379] send: files +2025-01-07 16:27:35,176 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json with policy now +2025-01-07 16:27:35,176 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:35,177 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:35,177 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:35,347 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:35,353 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/0c5b3dxk-media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json +2025-01-07 16:27:35,594 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/14edwdcm-media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json +2025-01-07 16:27:35,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:36,141 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:36,244 DEBUG SenderThread:2183860 [sender.py:send():379] send: exit +2025-01-07 16:27:36,244 INFO SenderThread:2183860 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 16:27:36,244 INFO SenderThread:2183860 [sender.py:send_exit():588] handling runtime: 488 +2025-01-07 16:27:36,244 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:36,244 INFO SenderThread:2183860 [sender.py:send_exit():594] send defer +2025-01-07 16:27:36,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,245 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 16:27:36,245 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:36,245 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 16:27:36,245 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 1 +2025-01-07 16:27:36,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,245 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 16:27:36,245 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:36,245 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 16:27:36,245 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 2 +2025-01-07 16:27:36,245 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,245 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 16:27:36,246 INFO HandlerThread:2183860 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 16:27:36,246 DEBUG SystemMonitor:2183860 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 16:27:36,246 DEBUG SystemMonitor:2183860 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 16:27:36,246 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 16:27:36,247 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 16:27:36,298 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 16:27:36,298 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 16:27:36,298 INFO HandlerThread:2183860 [interfaces.py:finish():200] Joined network monitor +2025-01-07 16:27:36,299 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:36,299 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 16:27:36,299 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 3 +2025-01-07 16:27:36,299 DEBUG SenderThread:2183860 [sender.py:send():379] send: stats +2025-01-07 16:27:36,299 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,299 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 16:27:36,299 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:36,299 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 16:27:36,299 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 4 +2025-01-07 16:27:36,300 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:36,300 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 16:27:36,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:36,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:36,938 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml +2025-01-07 16:27:42,856 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:42,856 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:42,856 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 16:27:42,857 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:42,857 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 16:27:42,857 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 5 +2025-01-07 16:27:42,857 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:42,857 DEBUG SenderThread:2183860 [sender.py:send():379] send: history +2025-01-07 16:27:42,857 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:27:42,858 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:42,858 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:42,858 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 16:27:42,859 DEBUG SenderThread:2183860 [sender.py:send():379] send: summary +2025-01-07 16:27:42,859 INFO SenderThread:2183860 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:27:42,859 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:42,859 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 16:27:42,859 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 6 +2025-01-07 16:27:42,859 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:42,859 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 16:27:42,859 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:42,859 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 16:27:42,860 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 7 +2025-01-07 16:27:42,860 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:27:42,860 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:42,860 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 16:27:42,860 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:42,860 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 16:27:42,940 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:43,183 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:44,931 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 8 +2025-01-07 16:27:44,931 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:44,931 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:44,931 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 16:27:44,931 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:44,931 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 16:27:44,931 INFO SenderThread:2183860 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 16:27:44,934 INFO SenderThread:2183860 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 16:27:44,936 INFO SenderThread:2183860 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 16:27:44,941 INFO Thread-12 :2183860 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:44,946 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 9 +2025-01-07 16:27:44,946 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:44,946 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 16:27:44,947 DEBUG SenderThread:2183860 [sender.py:send():379] send: artifact +2025-01-07 16:27:45,183 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:45,606 INFO wandb-upload_0:2183860 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp5_jfea2 +2025-01-07 16:27:45,816 INFO wandb-upload_1:2183860 [upload_job.py:push():88] Uploaded file /tmp/tmpe_ev39sb/wandb-job.json +2025-01-07 16:27:46,356 INFO SenderThread:2183860 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjY1NzQxMQ==', 'versionIndex': 430}}} +2025-01-07 16:27:46,356 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:46,356 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 16:27:46,356 INFO SenderThread:2183860 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 16:27:46,942 INFO SenderThread:2183860 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files +2025-01-07 16:27:46,942 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml config.yaml +2025-01-07 16:27:46,942 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json wandb-summary.json +2025-01-07 16:27:46,943 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/requirements.txt requirements.txt +2025-01-07 16:27:46,943 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:27:46,943 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-metadata.json wandb-metadata.json +2025-01-07 16:27:46,944 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log output.log +2025-01-07 16:27:46,948 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 16:27:46,948 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 media/videos/eval/1_eval_1_887535ba826c760b568a.mp4 +2025-01-07 16:27:46,949 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 media/videos/eval/4_eval_4_a8a7a2360b5bcbe4011d.mp4 +2025-01-07 16:27:46,949 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 media/videos/eval/0_eval_0_89262cab025e888a6f21.mp4 +2025-01-07 16:27:46,950 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 media/videos/eval/2_eval_2_de6316f5b26b7e9850a5.mp4 +2025-01-07 16:27:46,950 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 media/videos/eval/3_eval_3_295a37e1d2a179eff243.mp4 +2025-01-07 16:27:46,952 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json media/table/eval/total_success_rate_ci_6_6360f159e4e74e6b412e.table.json +2025-01-07 16:27:46,952 INFO SenderThread:2183860 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:27:46,952 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 10 +2025-01-07 16:27:46,952 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:46,952 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:46,952 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 16:27:46,953 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:46,953 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 16:27:46,953 INFO SenderThread:2183860 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:27:47,119 INFO wandb-upload_2:2183860 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/wandb-summary.json +2025-01-07 16:27:47,144 INFO wandb-upload_0:2183860 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/config.yaml +2025-01-07 16:27:47,170 INFO wandb-upload_4:2183860 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/output.log +2025-01-07 16:27:47,184 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:47,185 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:47,198 INFO wandb-upload_1:2183860 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/files/requirements.txt +2025-01-07 16:27:47,221 INFO wandb-upload_3:2183860 [upload_job.py:push():130] Uploaded file /tmp/tmpyeyiwcj4wandb/2t6nakx0-events.out.tfevents.1736284768.leonmkim-ROG-Strix-G15CS-G15CS.2183494.0 +2025-01-07 16:27:47,422 INFO Thread-11 (_thread_body):2183860 [sender.py:transition_state():614] send defer: 11 +2025-01-07 16:27:47,422 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:47,422 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 16:27:47,422 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:47,422 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 16:27:47,422 INFO SenderThread:2183860 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 16:27:47,424 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 12 +2025-01-07 16:27:47,424 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:47,424 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 16:27:47,424 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:47,424 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 16:27:47,424 INFO SenderThread:2183860 [file_stream.py:finish():601] file stream finish called +2025-01-07 16:27:47,600 INFO SenderThread:2183860 [file_stream.py:finish():605] file stream finish is done +2025-01-07 16:27:47,600 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 13 +2025-01-07 16:27:47,600 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:47,601 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 16:27:47,601 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:47,601 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 16:27:47,601 INFO SenderThread:2183860 [sender.py:transition_state():614] send defer: 14 +2025-01-07 16:27:47,601 DEBUG SenderThread:2183860 [sender.py:send():379] send: final +2025-01-07 16:27:47,601 DEBUG SenderThread:2183860 [sender.py:send():379] send: footer +2025-01-07 16:27:47,601 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:27:47,601 INFO HandlerThread:2183860 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 16:27:47,602 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: defer +2025-01-07 16:27:47,602 INFO SenderThread:2183860 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 16:27:47,602 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:47,602 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 16:27:47,602 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:27:47,603 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 16:27:47,603 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 16:27:47,604 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:47,604 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: server_info +2025-01-07 16:27:47,604 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:27:47,655 DEBUG SenderThread:2183860 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:27:47,655 INFO MainThread:2183860 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 16:27:47,656 INFO MainThread:2183860 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 16:27:47,656 INFO MainThread:2183860 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 16:27:47,657 DEBUG HandlerThread:2183860 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 16:27:47,657 INFO HandlerThread:2183860 [handler.py:finish():882] shutting down handler +2025-01-07 16:27:48,603 INFO WriterThread:2183860 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/run-kd9mh2vh.wandb +2025-01-07 16:27:48,655 INFO SenderThread:2183860 [sender.py:finish():1615] shutting down sender +2025-01-07 16:27:48,655 INFO SenderThread:2183860 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:27:48,655 INFO SenderThread:2183860 [file_pusher.py:join():175] waiting for file pusher diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug.log b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..454dfa4b768ad247770cb7dc5d5026ca04c9987a --- /dev/null +++ b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Configure stats pid to 2183494 +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/settings +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug.log +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/logs/debug-internal.log +2025-01-07 16:19:27,460 INFO MainThread:2183494 [wandb_init.py:init():569] calling init triggers +2025-01-07 16:19:27,461 INFO MainThread:2183494 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh'} +2025-01-07 16:19:27,461 INFO MainThread:2183494 [wandb_init.py:init():619] starting backend +2025-01-07 16:19:27,461 INFO MainThread:2183494 [wandb_init.py:init():623] setting up manager +2025-01-07 16:19:27,465 INFO MainThread:2183494 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 16:19:27,466 INFO MainThread:2183494 [wandb_init.py:init():631] backend started and connected +2025-01-07 16:19:27,479 INFO MainThread:2183494 [wandb_init.py:init():720] updated telemetry +2025-01-07 16:19:27,487 INFO MainThread:2183494 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 16:19:27,786 INFO MainThread:2183494 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 16:19:27,857 INFO MainThread:2183494 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 16:19:27,857 INFO MainThread:2183494 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 16:19:28,214 INFO MainThread:2183494 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 16:19:28,214 INFO MainThread:2183494 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 16:19:28,214 INFO MainThread:2183494 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 16:19:28,214 INFO MainThread:2183494 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 16:19:28,215 INFO MainThread:2183494 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 16:19:28,216 INFO MainThread:2183494 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kd9mh2vh/tb, True +2025-01-07 16:19:53,104 INFO MainThread:2183494 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 16:27:48,658 WARNING MsgRouterThr:2183494 [router.py:message_loop():77] message_loop has been closed diff --git a/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/run-kd9mh2vh.wandb b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/run-kd9mh2vh.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ffc6aafe4267feb1bf320516dc685e5c8b334215 Binary files /dev/null and b/kd9mh2vh/wandb/run-20250107_161927-kd9mh2vh/run-kd9mh2vh.wandb differ diff --git a/kkp1x4wb/episode_rosbags/aligned_depth_to_color_K.npy b/kkp1x4wb/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kkp1x4wb/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kkp1x4wb/episode_rosbags/cam_tf_world.npy b/kkp1x4wb/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/kkp1x4wb/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/kkp1x4wb/episode_rosbags/color_K.npy b/kkp1x4wb/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kkp1x4wb/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kkp1x4wb/episode_rosbags/depth_K.npy b/kkp1x4wb/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/kkp1x4wb/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/kkp1x4wb/eval_robot.log b/kkp1x4wb/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..1ace1161e807fef371b770ed51592e0941c3a765 --- /dev/null +++ b/kkp1x4wb/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 18:12:38,602][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 18:12:38,605][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/kkp1x4wb/wandb/debug-internal.log b/kkp1x4wb/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f4197119eaf5effae892c2ba374f422b9eb41850 --- /dev/null +++ b/kkp1x4wb/wandb/debug-internal.log @@ -0,0 +1,1089 @@ +2025-01-06 18:12:44,353 INFO StreamThr :2099720 [internal.py:wandb_internal():85] W&B internal server running at pid: 2099720, started at: 2025-01-06 18:12:44.353503 +2025-01-06 18:12:44,355 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status +2025-01-06 18:12:44,366 INFO WriterThread:2099720 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/run-kkp1x4wb.wandb +2025-01-06 18:12:44,367 DEBUG SenderThread:2099720 [sender.py:send():379] send: header +2025-01-06 18:12:44,377 DEBUG SenderThread:2099720 [sender.py:send():379] send: run +2025-01-06 18:12:44,646 INFO SenderThread:2099720 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files +2025-01-06 18:12:44,646 INFO SenderThread:2099720 [sender.py:_start_run_threads():1188] run started: kkp1x4wb with start time 1736205164.355427 +2025-01-06 18:12:44,663 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 18:12:44,664 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: check_version +2025-01-06 18:12:44,708 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 18:12:44,713 DEBUG HandlerThread:2099720 [system_info.py:__init__():26] System info init +2025-01-06 18:12:44,713 DEBUG HandlerThread:2099720 [system_info.py:__init__():41] System info init done +2025-01-06 18:12:44,713 INFO HandlerThread:2099720 [system_monitor.py:start():194] Starting system monitor +2025-01-06 18:12:44,713 INFO SystemMonitor:2099720 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 18:12:44,713 INFO HandlerThread:2099720 [system_monitor.py:probe():214] Collecting system info +2025-01-06 18:12:44,714 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 18:12:44,715 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started disk monitoring +2025-01-06 18:12:44,715 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 18:12:44,717 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started memory monitoring +2025-01-06 18:12:44,717 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started network monitoring +2025-01-06 18:12:44,727 DEBUG HandlerThread:2099720 [system_info.py:probe():152] Probing system +2025-01-06 18:12:44,729 DEBUG HandlerThread:2099720 [system_info.py:_probe_git():137] Probing git +2025-01-06 18:12:44,733 DEBUG HandlerThread:2099720 [system_info.py:_probe_git():145] Probing git done +2025-01-06 18:12:44,733 DEBUG HandlerThread:2099720 [system_info.py:probe():200] Probing system done +2025-01-06 18:12:44,733 DEBUG HandlerThread:2099720 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T23:12:44.727725', 'startedAt': '2025-01-06T23:12:44.345780', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.91925, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.968, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 413.9370460510254}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 18:12:44,734 INFO HandlerThread:2099720 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 18:12:44,734 INFO HandlerThread:2099720 [system_monitor.py:probe():227] Publishing system info +2025-01-06 18:12:44,734 DEBUG HandlerThread:2099720 [system_info.py:_save_code():44] Saving code +2025-01-06 18:12:44,735 DEBUG HandlerThread:2099720 [system_info.py:_save_code():67] Saving code done +2025-01-06 18:12:44,735 DEBUG HandlerThread:2099720 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 18:12:44,811 DEBUG HandlerThread:2099720 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 18:12:44,811 INFO HandlerThread:2099720 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 18:12:44,814 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:12:44,814 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 18:12:44,815 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 18:12:44,815 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 18:12:45,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 18:12:45,056 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: python_packages +2025-01-06 18:12:45,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:12:45,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:12:45,073 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/1n933w7g-code/FISH/eval_robot.py +2025-01-06 18:12:45,092 INFO wandb-upload_0:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/rrka7m8g-wandb-metadata.json +2025-01-06 18:12:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:45,101 INFO HandlerThread:2099720 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/tb" + save: true +} +control { + relay_id: "4f3b3a8842bf4649a7ba726a88e2c301" +} +_info { + stream_id: "kkp1x4wb" +} + +2025-01-06 18:12:45,101 DEBUG HandlerThread:2099720 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 18:12:45,112 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/l2wspz5x-diff.patch +2025-01-06 18:12:45,195 DEBUG SenderThread:2099720 [sender.py:send():379] send: telemetry +2025-01-06 18:12:45,195 DEBUG SenderThread:2099720 [sender.py:send():379] send: tbrecord +2025-01-06 18:12:45,196 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:12:45,196 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 with policy live +2025-01-06 18:12:45,371 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/0mg4syc6-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:12:45,648 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/code/FISH/eval_robot.py +2025-01-06 18:12:45,648 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/requirements.txt +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/diff.patch +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-metadata.json +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/code/FISH +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/code +2025-01-06 18:12:46,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:47,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:47,648 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:12:48,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:49,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:50,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:12:50,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:51,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:52,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:53,041 DEBUG SenderThread:2099720 [sender.py:send():379] send: config +2025-01-06 18:12:53,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:54,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:55,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:12:55,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:55,650 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:12:56,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:57,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:58,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:59,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:59,651 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:00,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:13:00,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:13:00,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:00,119 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:01,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:01,651 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:02,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:03,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:04,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:05,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:06,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:06,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:07,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:08,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:09,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:10,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:11,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:11,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:12,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:13,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:14,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:15,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:13:15,056 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:13:15,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:16,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:16,147 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:16,656 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml +2025-01-06 18:13:17,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:18,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:19,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:20,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:21,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:22,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:22,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:23,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:24,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:25,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:26,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:27,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:27,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:28,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:29,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:30,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:13:30,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:13:30,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:31,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:32,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:32,138 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:33,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:34,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:35,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:35,661 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:36,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:37,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:37,528 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:37,661 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:38,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:39,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:39,662 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:40,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:41,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:42,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:43,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:43,066 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:44,066 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:44,718 DEBUG SystemMonitor:2099720 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 18:13:44,718 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:13:45,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:13:45,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:13:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:46,066 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:47,066 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:48,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:48,152 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:49,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:50,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:51,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:52,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:53,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:54,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:54,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:55,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:55,666 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:56,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:57,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:58,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:59,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:59,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:00,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:14:00,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:14:00,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:01,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:02,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:03,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:04,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:04,193 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:05,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:06,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:07,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:08,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:09,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:10,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:10,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:11,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:12,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:13,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:13,294 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:14:13,294 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 with policy now +2025-01-06 18:14:13,295 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:14:13,296 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:14:13,297 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:14:13,299 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:14:13,523 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/t24i6gkl-media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 +2025-01-06 18:14:13,671 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:14:13,671 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 +2025-01-06 18:14:13,671 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:14:13,671 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos +2025-01-06 18:14:13,672 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:14:13,672 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media +2025-01-06 18:14:13,941 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/jnrwz3kr-media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 +2025-01-06 18:14:14,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:14,719 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:14:15,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:14:15,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:14:15,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:15,187 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:15,672 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:14:16,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:17,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:18,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:19,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:19,673 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:14:20,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:21,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:21,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:21,674 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:14:22,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:23,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:24,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:25,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:26,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:26,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:27,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:28,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:29,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:30,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:14:30,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:14:30,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:31,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:31,212 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:32,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:33,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:34,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:35,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:36,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:37,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:37,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:38,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:39,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:40,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:41,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:42,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:42,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:43,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:44,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:44,719 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:14:45,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:14:45,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:14:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:46,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:47,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:47,202 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:48,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:49,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:50,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:51,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:52,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:53,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:53,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:54,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:55,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:56,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:57,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:58,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:59,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:59,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:00,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:15:00,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:15:00,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:01,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:02,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:03,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:04,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:04,140 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:05,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:06,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:07,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:08,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:09,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:10,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:10,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:11,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:12,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:13,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:14,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:14,720 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:15:15,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:15:15,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:15:15,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:15,130 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:16,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:17,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:18,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:19,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:20,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:21,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:21,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:22,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:23,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:24,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:25,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:26,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:26,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:27,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:28,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:29,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:30,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:15:30,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:15:30,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:31,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:31,153 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:32,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:33,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:34,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:35,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:36,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:37,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:37,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:38,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:39,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:40,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:41,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:42,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:42,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:43,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:44,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:44,720 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:15:45,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:15:45,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:15:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:46,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:47,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:47,141 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:48,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:49,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:50,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:51,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:52,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:53,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:53,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:54,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:55,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:56,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:57,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:58,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:59,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:59,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:00,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:16:00,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:16:00,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:01,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:02,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:03,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:04,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:04,133 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:05,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:06,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:07,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:08,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:09,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:10,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:10,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:11,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:12,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:13,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:14,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:14,721 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:16:15,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:16:15,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:16:15,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:15,155 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:16,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:17,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:18,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:19,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:20,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:21,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:21,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:22,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:23,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:24,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:25,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:26,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:26,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:27,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:28,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:29,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:30,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:16:30,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:16:30,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:31,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:31,138 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:32,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:33,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:34,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:35,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:36,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:37,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:37,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:38,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:39,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:40,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:41,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:42,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:42,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:43,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:44,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:44,721 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:16:45,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:16:45,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:16:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:46,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:47,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:47,209 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:48,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:49,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:50,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:51,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:52,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:53,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:53,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:54,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:55,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:56,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:57,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:58,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:58,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:59,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:00,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:17:00,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:17:00,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:01,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:02,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:03,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:03,214 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:04,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:05,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:06,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:07,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:08,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:09,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:09,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:10,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:11,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:12,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:13,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:14,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:14,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:14,721 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:17:15,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:17:15,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:17:15,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:16,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:17,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:18,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:19,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:19,210 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:20,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:21,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:22,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:23,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:24,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:25,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:25,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:26,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:27,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:28,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:29,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:29,738 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:29,738 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:17:29,986 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/7mqcmgat-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:17:30,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:17:30,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:17:30,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:30,182 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:31,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:31,738 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:32,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:33,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:33,739 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:34,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:35,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:36,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:36,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:37,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:38,100 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:39,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:40,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:41,100 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:41,177 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:41,742 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:42,100 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:43,100 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:44,103 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:44,722 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:17:45,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:17:45,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:17:45,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:46,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:47,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:47,163 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:48,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:49,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:50,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:51,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:52,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:52,169 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:53,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:53,745 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:54,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:55,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:56,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:57,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:58,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:58,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:59,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:00,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:18:00,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:18:00,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:01,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:02,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:03,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:03,152 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:04,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:05,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:06,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:07,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:08,107 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:08,297 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:08,617 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:18:08,617 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 with policy now +2025-01-06 18:18:08,618 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:18:08,618 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:18:08,619 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:18:08,619 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:18:08,750 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:18:08,750 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 +2025-01-06 18:18:08,750 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:18:08,839 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/9bs6626v-media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 +2025-01-06 18:18:09,100 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/873d9gvr-media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 +2025-01-06 18:18:09,109 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:09,750 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:10,111 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:11,110 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:12,121 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:13,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:13,751 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:14,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:14,197 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:14,722 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:18:15,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:18:15,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:18:15,111 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:15,752 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:16,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:17,111 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:18,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:19,111 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:20,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:20,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:20,759 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:18:20,764 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:18:20,764 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:18:20,764 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:18:21,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:21,759 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:21,759 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:18:22,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:23,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:23,760 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:24,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:25,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:25,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:25,761 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:26,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:27,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:28,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:29,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:30,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:18:30,060 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:18:30,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:30,167 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:31,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:32,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:33,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:34,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:35,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:36,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:36,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:37,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:38,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:39,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:40,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:41,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:41,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:42,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:43,115 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:44,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:44,723 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:18:45,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:18:45,060 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:18:45,115 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:46,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:46,197 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:47,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:47,768 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:48,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:49,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:50,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:51,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:51,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:51,769 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:52,117 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:53,117 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:54,117 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:55,117 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:56,118 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:56,749 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:57,118 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:57,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:58,118 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:59,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:59,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:00,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:19:00,063 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:19:00,209 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:19:00,210 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:19:00,210 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 with policy now +2025-01-06 18:19:00,210 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:19:00,210 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:19:00,211 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:00,227 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:19:00,463 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/kkdndppd-media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 +2025-01-06 18:19:00,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:19:00,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 +2025-01-06 18:19:00,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:19:01,040 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/h7baghxu-media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 +2025-01-06 18:19:01,212 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:01,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:02,215 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:02,265 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:02,773 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:03,213 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:04,217 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:05,219 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:05,774 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:06,216 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:07,217 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:07,665 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:07,775 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:08,220 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:09,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:10,218 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:11,221 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:12,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:13,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:13,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:14,222 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:14,723 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:19:15,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:19:15,063 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:19:15,224 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:16,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:17,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:17,627 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:19:17,627 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:19:17,627 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:19:17,778 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:17,778 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:19:17,778 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:19:18,027 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/ecwvmssm-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:19:18,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:19,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:19,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:19,778 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:20,224 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:21,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:21,779 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:22,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:23,226 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:24,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:24,224 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:25,224 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:26,232 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:27,229 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:28,229 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:29,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:30,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:19:30,061 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:19:30,152 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:30,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:31,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:32,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:33,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:34,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:35,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:36,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:36,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:37,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:38,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:39,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:40,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:41,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:41,232 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:42,232 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:43,232 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:44,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:44,724 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:19:45,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:19:45,061 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:19:45,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:46,126 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:46,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:47,234 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:48,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:49,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:50,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:51,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:52,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:52,234 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:53,234 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:54,234 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:55,235 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:56,236 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:56,966 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:19:56,966 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 with policy now +2025-01-06 18:19:56,969 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:19:56,969 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:19:56,969 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:19:57,004 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:19:57,241 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:57,242 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/2g0e0a4w-media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 +2025-01-06 18:19:57,791 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:57,791 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:19:57,792 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 +2025-01-06 18:19:57,792 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:19:58,029 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:58,069 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/0ie7p8hp-media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 +2025-01-06 18:19:58,255 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:58,792 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:59,244 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:00,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:20:00,069 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:20:00,245 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:01,249 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:01,793 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:02,250 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:03,250 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:03,458 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:04,256 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:05,254 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:05,794 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:06,257 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:07,253 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:08,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:09,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:09,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:10,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:11,256 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:12,255 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:12,720 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:20:12,720 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:20:12,721 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:20:12,796 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:20:12,796 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:20:13,258 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:13,797 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:14,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:14,724 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:20:14,724 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:15,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:20:15,070 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:20:15,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:16,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:17,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:17,798 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:18,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:19,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:20,048 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:20,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:21,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:22,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:23,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:24,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:25,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:25,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:26,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:27,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:28,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:29,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:30,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:20:30,062 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:20:30,184 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:30,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:31,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:32,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:33,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:34,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:35,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:36,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:36,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:37,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:38,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:39,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:40,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:41,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:41,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:42,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:43,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:44,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:44,725 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:20:45,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:20:45,062 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:20:45,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:46,200 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:46,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:47,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:48,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:49,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:50,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:51,275 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:51,279 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:51,809 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:51,809 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 +2025-01-06 18:20:51,810 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:20:51,822 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:20:51,822 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 with policy now +2025-01-06 18:20:51,825 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:20:51,826 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:20:51,826 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:20:51,826 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:20:52,033 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/wrnj6d2q-media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 +2025-01-06 18:20:52,277 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:52,294 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/pqvst0yk-media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 +2025-01-06 18:20:52,810 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:20:53,282 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:53,810 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:54,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:54,810 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:55,289 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:56,283 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:56,521 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:57,301 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:57,811 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:58,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:59,294 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:59,803 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:20:59,803 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:20:59,803 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:20:59,811 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 18:20:59,811 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 18:20:59,812 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:59,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:20:59,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:00,016 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:21:00,052 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/5e1oia4n-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:00,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:21:00,286 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:00,345 INFO wandb-upload_1:2099720 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfjqaqtbv +2025-01-06 18:21:00,387 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 18:21:00,395 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:21:00,753 INFO SenderThread:2099720 [sender.py:send_request_log_artifact():1525] logged artifact run-kkp1x4wb-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjAwNDA0NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxOTYwMg==', 'latestArtifact': None}} +2025-01-06 18:21:00,753 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:21:00,753 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 18:21:00,754 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:21:00,754 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,755 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,755 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:21:00,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:00,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:00,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-06 18:21:00,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:21:00,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media +2025-01-06 18:21:00,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table +2025-01-06 18:21:00,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval +2025-01-06 18:21:00,853 DEBUG SenderThread:2099720 [sender.py:send():379] send: telemetry +2025-01-06 18:21:00,853 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,853 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,853 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,854 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,854 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,854 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,854 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,855 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,855 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,855 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,855 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,855 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,856 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,856 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,856 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,856 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,857 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,857 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,857 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,857 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,857 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,858 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,858 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 18:21:00,935 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/uwgcvwwj-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:21:01,036 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/ccuehux8-media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-06 18:21:01,188 INFO wandb-upload_0:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/vgccyunv-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:21:01,286 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:01,427 INFO wandb-upload_2:2099720 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5nz775i3 +2025-01-06 18:21:01,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:21:01,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:01,829 INFO SenderThread:2099720 [sender.py:send_request_log_artifact():1525] logged artifact run-kkp1x4wb-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjAwNDA3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxOTYwMw==', 'latestArtifact': None}} +2025-01-06 18:21:01,830 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:21:01,830 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json with policy now +2025-01-06 18:21:01,830 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:21:01,830 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:21:01,830 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:01,831 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:02,072 INFO wandb-upload_0:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/ih1ud28a-media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-06 18:21:02,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:02,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:03,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:04,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:05,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:06,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:07,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:21:07,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:07,717 DEBUG SenderThread:2099720 [sender.py:send():379] send: exit +2025-01-06 18:21:07,717 INFO SenderThread:2099720 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 18:21:07,717 INFO SenderThread:2099720 [sender.py:send_exit():588] handling runtime: 503 +2025-01-06 18:21:07,718 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:07,718 INFO SenderThread:2099720 [sender.py:send_exit():594] send defer +2025-01-06 18:21:07,718 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,718 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 18:21:07,718 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:07,718 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 18:21:07,718 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 1 +2025-01-06 18:21:07,718 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,718 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 18:21:07,719 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:07,719 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 18:21:07,719 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 2 +2025-01-06 18:21:07,719 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,719 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 18:21:07,719 INFO HandlerThread:2099720 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 18:21:07,719 DEBUG SystemMonitor:2099720 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 18:21:07,719 DEBUG SystemMonitor:2099720 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 18:21:07,719 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 18:21:07,720 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 18:21:07,771 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 18:21:07,772 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 18:21:07,772 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined network monitor +2025-01-06 18:21:07,772 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:07,772 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 18:21:07,772 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 3 +2025-01-06 18:21:07,772 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:21:07,773 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,773 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 18:21:07,773 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:07,773 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 18:21:07,773 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 4 +2025-01-06 18:21:07,773 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,773 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 18:21:07,816 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:09,817 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:21:13,808 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:13,808 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:21:13,808 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 18:21:13,810 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:13,810 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 18:21:13,810 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 5 +2025-01-06 18:21:13,810 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:13,810 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:21:13,810 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:13,810 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:13,810 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 18:21:13,811 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:13,811 DEBUG SenderThread:2099720 [sender.py:send():379] send: summary +2025-01-06 18:21:13,811 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:13,812 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:13,812 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 18:21:13,812 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 6 +2025-01-06 18:21:13,812 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:13,812 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 18:21:13,812 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:13,812 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 18:21:13,815 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:21:13,818 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:13,952 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 7 +2025-01-06 18:21:13,952 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:13,952 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 18:21:13,952 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:13,952 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 18:21:14,718 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:14,819 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml +2025-01-06 18:21:16,337 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 8 +2025-01-06 18:21:16,337 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:16,337 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:16,337 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 18:21:16,338 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:16,338 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 18:21:16,338 INFO SenderThread:2099720 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 18:21:16,340 INFO SenderThread:2099720 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 18:21:16,342 INFO SenderThread:2099720 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 18:21:16,355 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 9 +2025-01-06 18:21:16,356 DEBUG SenderThread:2099720 [sender.py:send():379] send: artifact +2025-01-06 18:21:16,356 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:16,359 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 18:21:16,572 INFO SenderThread:2099720 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 18:21:16,572 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:16,572 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 18:21:16,572 INFO SenderThread:2099720 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 18:21:16,719 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:16,820 INFO SenderThread:2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:21:16,820 INFO SenderThread:2099720 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files +2025-01-06 18:21:16,820 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml config.yaml +2025-01-06 18:21:16,821 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json wandb-summary.json +2025-01-06 18:21:16,821 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/requirements.txt requirements.txt +2025-01-06 18:21:16,823 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-metadata.json wandb-metadata.json +2025-01-06 18:21:16,823 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:16,823 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log output.log +2025-01-06 18:21:16,823 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/diff.patch diff.patch +2025-01-06 18:21:16,826 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 10 +2025-01-06 18:21:16,827 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:16,829 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:16,830 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 18:21:16,830 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:16,830 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 18:21:16,831 INFO SenderThread:2099720 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 18:21:16,996 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/requirements.txt +2025-01-06 18:21:17,035 INFO wandb-upload_0:2099720 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:17,049 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml +2025-01-06 18:21:17,101 INFO wandb-upload_4:2099720 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:21:17,111 INFO wandb-upload_3:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/78bt69nu-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:17,311 INFO Thread-11 (_thread_body):2099720 [sender.py:transition_state():614] send defer: 11 +2025-01-06 18:21:17,312 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:17,312 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 18:21:17,312 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:17,312 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 18:21:17,312 INFO SenderThread:2099720 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 18:21:17,313 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 12 +2025-01-06 18:21:17,313 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:17,313 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 18:21:17,314 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:17,314 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 18:21:17,314 INFO SenderThread:2099720 [file_stream.py:finish():601] file stream finish called +2025-01-06 18:21:17,390 INFO SenderThread:2099720 [file_stream.py:finish():605] file stream finish is done +2025-01-06 18:21:17,390 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 13 +2025-01-06 18:21:17,390 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:17,390 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 18:21:17,390 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:17,390 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 18:21:17,390 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 14 +2025-01-06 18:21:17,390 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:17,390 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 18:21:17,391 DEBUG SenderThread:2099720 [sender.py:send():379] send: final +2025-01-06 18:21:17,391 DEBUG SenderThread:2099720 [sender.py:send():379] send: footer +2025-01-06 18:21:17,391 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:17,391 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 18:21:17,391 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:17,392 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 18:21:17,392 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:17,392 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:17,392 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: server_info +2025-01-06 18:21:17,392 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 18:21:17,394 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 18:21:17,395 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:17,458 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:17,458 INFO MainThread:2099720 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 18:21:17,458 INFO MainThread:2099720 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 18:21:17,459 INFO MainThread:2099720 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 18:21:17,459 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 18:21:17,459 INFO HandlerThread:2099720 [handler.py:finish():882] shutting down handler +2025-01-06 18:21:18,395 INFO WriterThread:2099720 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/run-kkp1x4wb.wandb +2025-01-06 18:21:18,458 INFO SenderThread:2099720 [sender.py:finish():1615] shutting down sender +2025-01-06 18:21:18,458 INFO SenderThread:2099720 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 18:21:18,458 INFO SenderThread:2099720 [file_pusher.py:join():175] waiting for file pusher diff --git a/kkp1x4wb/wandb/debug.log b/kkp1x4wb/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6216f865ffab169fe8aebdcfbe5bc7ef35246b0f --- /dev/null +++ b/kkp1x4wb/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 18:12:44,347 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Configure stats pid to 2099546 +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/settings +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug.log +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug-internal.log +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_init.py:init():569] calling init triggers +2025-01-06 18:12:44,349 INFO MainThread:2099546 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb'} +2025-01-06 18:12:44,349 INFO MainThread:2099546 [wandb_init.py:init():619] starting backend +2025-01-06 18:12:44,349 INFO MainThread:2099546 [wandb_init.py:init():623] setting up manager +2025-01-06 18:12:44,352 INFO MainThread:2099546 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 18:12:44,355 INFO MainThread:2099546 [wandb_init.py:init():631] backend started and connected +2025-01-06 18:12:44,366 INFO MainThread:2099546 [wandb_init.py:init():720] updated telemetry +2025-01-06 18:12:44,374 INFO MainThread:2099546 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 18:12:44,663 INFO MainThread:2099546 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 18:12:44,703 INFO MainThread:2099546 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 18:12:44,703 INFO MainThread:2099546 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 18:12:45,057 INFO MainThread:2099546 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 18:12:45,057 INFO MainThread:2099546 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 18:12:45,057 INFO MainThread:2099546 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 18:12:45,057 INFO MainThread:2099546 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 18:12:45,058 INFO MainThread:2099546 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 18:12:45,058 INFO MainThread:2099546 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/tb, True +2025-01-06 18:12:53,040 INFO MainThread:2099546 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 18:21:18,460 WARNING MsgRouterThr:2099546 [router.py:message_loop():77] message_loop has been closed diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e820e17fdc33b2bc09c926a57b795fa268f8e6f --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736205164 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/diff.patch b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..4c1a5f92eb09cddbc2402d3534362390d486902d --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..54e6cd0 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..cce0f428f851722b3cb84f2174378d6996531209 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[10, 10, 1.0, 0.6597032546677664, 1, 0.7793639599718221, 1, 0.8272221683801428, 1, 0.8133141546814853, 1]]} \ No newline at end of file diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0da22a4b06e98e801490165d0243fe12114978d --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9467d3a3a29eb4dd688af4152a447bfdbf46f7761c65a10619d6ec399b40eedf +size 1506021 diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b596dd5aa4e95f6b4c496a3e078133fcb0e4d812 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6baff3ebaf384ea295b495aba2801497484e69a3618a9795538f7e8876f74179 +size 1433492 diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4981f728a68282808ea3293b508197fa9e4b6885 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:426b092ff928a3231ed10e53eb58d4e58bd0689388c55bb2f707e6991f9f4d10 +size 1451506 diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c4d3cac89db6e8571b6c2d238e51ca189936be73 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b5ff732fe53abaf1668adba06190d94b9287725af2561047a7e66e140203e89 +size 1432708 diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b3a204393b5be71f1f095879de322c423610bafe --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3009e51676e13f8bfc1c7d5ef68ebacb8c4c8fd852bd776bf16fd6b19d03119a +size 1441758 diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..50d67cd8ce91bf1a1d9655056d1eb9e5eef2e1d4 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log @@ -0,0 +1,271 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736205173.041040]: resetting environment +[INFO] [1736205173.045602]: cleared current plan +[INFO] [1736205173.046658]: moving to home +[INFO] [1736205174.048498]: reached home +[INFO] [1736205174.049726]: reset action history +[INFO] [1736205176.760873]: environment reset +Starting episode 0 +[INFO] [1736205176.765994]: resetting environment +[INFO] [1736205176.773520]: cleared current plan +[INFO] [1736205176.774437]: moving to home +[INFO] [1736205177.776510]: reached home +[INFO] [1736205177.777815]: reset action history +[INFO] [1736205180.487793]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736205213.807772]: resetting environment +[INFO] [1736205213.814724]: cleared current plan +[INFO] [1736205213.815808]: moving to home +[INFO] [1736205214.818500]: reached home +[INFO] [1736205214.819871]: reset action history +[INFO] [1736205217.527992]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/episode_rosbags/episode_0_2025-01-06-18-13-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2099546.log +started roslaunch server http://158.130.50.37:35855/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2099828] +process[print_text-2]: started with pid [2099852] +process[rosbag_record-3]: started with pid [2099853] +started bagging! +For topic gripper_width: timestamp difference is 108424380 for nearest: 1736205234406784251 - target: 1736205234298359871 at idx 220 +[INFO] [1736205252.353525]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736205253.083080]: Stored episode 1. +[INFO] [1736205253.083339]: Saving video... +[INFO] [1736205253.284614]: Video saved! +Starting episode 1 +[INFO] [1736205253.297348]: resetting environment +[INFO] [1736205253.315094]: cleared current plan +[INFO] [1736205253.315371]: moving to home +[INFO] [1736205256.922956]: reached home +[INFO] [1736205256.923441]: reset action history +[INFO] [1736205259.641635]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736205448.929234]: resetting environment +[INFO] [1736205448.932479]: cleared current plan +[INFO] [1736205448.932657]: moving to home +[INFO] [1736205449.934321]: reached home +[INFO] [1736205449.934896]: reset action history +[INFO] [1736205452.642098]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/episode_rosbags/episode_1_2025-01-06-18-17-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2099546.log +started roslaunch server http://158.130.50.37:41959/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2100534] +process[print_text-5]: started with pid [2100535] +process[rosbag_record-6]: started with pid [2100536] +started bagging! +For topic gripper_width: timestamp difference is -109285970 for nearest: 1736205459940095763 - target: 1736205460049381733 at idx 93 +For topic gripper_width: timestamp difference is 100509892 for nearest: 1736205472106763850 - target: 1736205472006253958 at idx 249 +[INFO] [1736205487.569308]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736205488.296556]: Stored episode 2. +[INFO] [1736205488.296762]: Saving video... +[INFO] [1736205488.593286]: Video saved! +Starting episode 2 +[INFO] [1736205488.619925]: resetting environment +[INFO] [1736205488.645407]: cleared current plan +[INFO] [1736205488.646368]: moving to home +[INFO] [1736205492.154192]: reached home +[INFO] [1736205492.159179]: reset action history +[INFO] [1736205494.890551]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736205500.183255]: resetting environment +[INFO] [1736205500.203854]: cleared current plan +[INFO] [1736205500.210378]: moving to home +[INFO] [1736205501.212216]: reached home +[INFO] [1736205501.212691]: reset action history +[INFO] [1736205503.921854]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/episode_rosbags/episode_2_2025-01-06-18-18-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2099546.log +started roslaunch server http://158.130.50.37:37029/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2100713] +process[print_text-8]: started with pid [2100714] +process[rosbag_record-9]: started with pid [2100738] +started bagging! +For topic gripper_width: timestamp difference is 111348258 for nearest: 1736205526806786166 - target: 1736205526695437908 at idx 249 +For topic gripper_width: timestamp difference is 114050427 for nearest: 1736205530206762591 - target: 1736205530092712164 at idx 249 +For topic gripper_width: timestamp difference is 118552010 for nearest: 1736205535706788819 - target: 1736205535588236809 at idx 249 +[INFO] [1736205538.874752]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736205539.701410]: Stored episode 3. +[INFO] [1736205539.703448]: Saving video... +[INFO] [1736205540.165434]: Video saved! +Starting episode 3 +[INFO] [1736205540.205639]: resetting environment +[INFO] [1736205540.220939]: cleared current plan +[INFO] [1736205540.221402]: moving to home +[WARN] [1736205540.193176]: Plan exhausted +[INFO] [1736205543.825307]: reached home +[INFO] [1736205543.830596]: reset action history +[INFO] [1736205546.617216]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736205557.004733]: resetting environment +[INFO] [1736205557.025670]: cleared current plan +[INFO] [1736205557.026911]: moving to home +[INFO] [1736205558.029214]: reached home +[INFO] [1736205558.032041]: reset action history +[INFO] [1736205560.807075]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/episode_rosbags/episode_3_2025-01-06-18-19-20.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2099546.log +started roslaunch server http://158.130.50.37:38539/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2100866] +process[print_text-11]: started with pid [2100867] +process[rosbag_record-12]: started with pid [2100892] +started bagging! +[INFO] [1736205595.652690]: Storing episode... +[pub_text-10] killing on exit +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1736205596.401597]: Stored episode 4. +[INFO] [1736205596.403604]: Saving video... +[INFO] [1736205596.916771]: Video saved! +Starting episode 4 +[INFO] [1736205596.971890]: resetting environment +[INFO] [1736205597.011924]: cleared current plan +[INFO] [1736205597.024100]: moving to home +[WARN] [1736205596.989702]: Plan exhausted +[INFO] [1736205600.638423]: reached home +[INFO] [1736205600.650673]: reset action history +[INFO] [1736205603.455475]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736205611.970966]: resetting environment +[INFO] [1736205611.990447]: cleared current plan +[INFO] [1736205611.990690]: moving to home +[INFO] [1736205612.993800]: reached home +[INFO] [1736205612.997014]: reset action history +[INFO] [1736205615.779155]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/episode_rosbags/episode_4_2025-01-06-18-20-15.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2099546.log +started roslaunch server http://158.130.50.37:43715/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2101019] +process[print_text-14]: started with pid [2101020] +process[rosbag_record-15]: started with pid [2101044] +started bagging! +[INFO] [1736205650.514179]: Storing episode... +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[INFO] [1736205651.278421]: Stored episode 5. +[INFO] [1736205651.282398]: Saving video... +[INFO] [1736205651.751233]: Video saved! +[INFO] [1736205651.827036]: resetting environment +[INFO] [1736205651.864889]: cleared current plan +[INFO] [1736205651.870611]: moving to home +[WARN] [1736205651.829747]: Plan exhausted +[INFO] [1736205655.477646]: reached home +[INFO] [1736205655.492335]: reset action history +[INFO] [1736205658.303367]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/requirements.txt b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-metadata.json b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3c8f71a81b7182561c6b4a1c46502a0c43cb00d7 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-06T23:12:44.727725", + "startedAt": "2025-01-06T23:12:44.345780", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3935.91925, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.996, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.968, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 413.9370460510254 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6575414fc3a375787eb80b2ea69b2241f1c113a4 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "9467d3a3a29eb4dd688af4152a447bfdbf46f7761c65a10619d6ec399b40eedf", "size": 1506021, "path": "media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4"}, "global_step": 665, "_timestamp": 1736205660.3077796, "_runtime": 495.9523525238037, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "6baff3ebaf384ea295b495aba2801497484e69a3618a9795538f7e8876f74179", "size": 1433492, "path": "media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4"}, "eval/num_success": 5.0, "episode": 10.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "426b092ff928a3231ed10e53eb58d4e58bd0689388c55bb2f707e6991f9f4d10", "size": 1451506, "path": "media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "6b5ff732fe53abaf1668adba06190d94b9287725af2561047a7e66e140203e89", "size": 1432708, "path": "media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "3009e51676e13f8bfc1c7d5ef68ebacb8c4c8fd852bd776bf16fd6b19d03119a", "size": 1441758, "path": "media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://o0z4mbvob857ynzvi23ajeoe0gnurackcca1ru27m8yv9fsl4n83veti5uakjnzxpqaa6hxu83xh8f19rpf5fi3c8f47jsi041wnp6bm9h1a6ujnqs72yfcu43exzcke/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://7nmrlwfajsnjw7udf7j2syx7n4xbiz9h4pc2rted2nogg0ujs6z7q1o8vpndreeqh8g9lfjwm56nov95dvjy4agua7i3ggfmk4qwwbg1nf2h3aie58444uhmuhm542zz:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 10, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.6597032546677664, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.7793639599718221, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.8272221683801428, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.8133141546814853, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "1e52f8fa0b3eaff8361f37df35c643ca7416e4a403526dc0f1ba264fbef828e8", "size": 437, "artifact_path": "wandb-client-artifact://61krk7ro9zels0m111fmkxn1lnmm1fb3w87b7bnvqe5j6rsgdsaan99lbqt2cl1il1u1rrxshc0yspxx0jcy6c3h2jjzg6fjj2sjmsddmktw1xkazcxkh9rsa88w3ps6/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://aticeia6krv8fj1f51tfnzkf5rzrcautdi5vhkhfv5lgjeyev9f0tl8g422fvty34ci4ze6su1nxvcdnvxxw8623hzcyb79n3se6eyetqvi4mg9tmncm7jgnuihxut6k:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 503}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.659703254699707, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.7793639302253723, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.8272221684455872, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.8133141398429871, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug-internal.log b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f4197119eaf5effae892c2ba374f422b9eb41850 --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug-internal.log @@ -0,0 +1,1089 @@ +2025-01-06 18:12:44,353 INFO StreamThr :2099720 [internal.py:wandb_internal():85] W&B internal server running at pid: 2099720, started at: 2025-01-06 18:12:44.353503 +2025-01-06 18:12:44,355 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status +2025-01-06 18:12:44,366 INFO WriterThread:2099720 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/run-kkp1x4wb.wandb +2025-01-06 18:12:44,367 DEBUG SenderThread:2099720 [sender.py:send():379] send: header +2025-01-06 18:12:44,377 DEBUG SenderThread:2099720 [sender.py:send():379] send: run +2025-01-06 18:12:44,646 INFO SenderThread:2099720 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files +2025-01-06 18:12:44,646 INFO SenderThread:2099720 [sender.py:_start_run_threads():1188] run started: kkp1x4wb with start time 1736205164.355427 +2025-01-06 18:12:44,663 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 18:12:44,664 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: check_version +2025-01-06 18:12:44,708 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 18:12:44,713 DEBUG HandlerThread:2099720 [system_info.py:__init__():26] System info init +2025-01-06 18:12:44,713 DEBUG HandlerThread:2099720 [system_info.py:__init__():41] System info init done +2025-01-06 18:12:44,713 INFO HandlerThread:2099720 [system_monitor.py:start():194] Starting system monitor +2025-01-06 18:12:44,713 INFO SystemMonitor:2099720 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 18:12:44,713 INFO HandlerThread:2099720 [system_monitor.py:probe():214] Collecting system info +2025-01-06 18:12:44,714 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 18:12:44,715 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started disk monitoring +2025-01-06 18:12:44,715 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 18:12:44,717 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started memory monitoring +2025-01-06 18:12:44,717 INFO SystemMonitor:2099720 [interfaces.py:start():188] Started network monitoring +2025-01-06 18:12:44,727 DEBUG HandlerThread:2099720 [system_info.py:probe():152] Probing system +2025-01-06 18:12:44,729 DEBUG HandlerThread:2099720 [system_info.py:_probe_git():137] Probing git +2025-01-06 18:12:44,733 DEBUG HandlerThread:2099720 [system_info.py:_probe_git():145] Probing git done +2025-01-06 18:12:44,733 DEBUG HandlerThread:2099720 [system_info.py:probe():200] Probing system done +2025-01-06 18:12:44,733 DEBUG HandlerThread:2099720 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T23:12:44.727725', 'startedAt': '2025-01-06T23:12:44.345780', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.91925, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.968, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 413.9370460510254}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 18:12:44,734 INFO HandlerThread:2099720 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 18:12:44,734 INFO HandlerThread:2099720 [system_monitor.py:probe():227] Publishing system info +2025-01-06 18:12:44,734 DEBUG HandlerThread:2099720 [system_info.py:_save_code():44] Saving code +2025-01-06 18:12:44,735 DEBUG HandlerThread:2099720 [system_info.py:_save_code():67] Saving code done +2025-01-06 18:12:44,735 DEBUG HandlerThread:2099720 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 18:12:44,811 DEBUG HandlerThread:2099720 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 18:12:44,811 INFO HandlerThread:2099720 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 18:12:44,814 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:12:44,814 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 18:12:44,815 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 18:12:44,815 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 18:12:45,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 18:12:45,056 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: python_packages +2025-01-06 18:12:45,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:12:45,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:12:45,073 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/1n933w7g-code/FISH/eval_robot.py +2025-01-06 18:12:45,092 INFO wandb-upload_0:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/rrka7m8g-wandb-metadata.json +2025-01-06 18:12:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:45,101 INFO HandlerThread:2099720 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/tb" + save: true +} +control { + relay_id: "4f3b3a8842bf4649a7ba726a88e2c301" +} +_info { + stream_id: "kkp1x4wb" +} + +2025-01-06 18:12:45,101 DEBUG HandlerThread:2099720 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 18:12:45,112 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/l2wspz5x-diff.patch +2025-01-06 18:12:45,195 DEBUG SenderThread:2099720 [sender.py:send():379] send: telemetry +2025-01-06 18:12:45,195 DEBUG SenderThread:2099720 [sender.py:send():379] send: tbrecord +2025-01-06 18:12:45,196 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:12:45,196 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 with policy live +2025-01-06 18:12:45,371 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/0mg4syc6-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:12:45,648 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/code/FISH/eval_robot.py +2025-01-06 18:12:45,648 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/requirements.txt +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/diff.patch +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-metadata.json +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/code/FISH +2025-01-06 18:12:45,649 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/code +2025-01-06 18:12:46,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:47,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:47,648 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:12:48,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:49,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:50,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:12:50,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:51,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:52,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:53,041 DEBUG SenderThread:2099720 [sender.py:send():379] send: config +2025-01-06 18:12:53,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:54,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:55,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:12:55,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:55,650 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:12:56,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:57,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:58,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:59,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:12:59,651 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:00,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:13:00,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:13:00,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:00,119 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:01,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:01,651 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:02,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:03,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:04,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:05,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:06,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:06,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:07,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:08,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:09,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:10,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:11,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:11,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:12,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:13,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:14,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:15,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:13:15,056 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:13:15,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:16,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:16,147 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:16,656 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml +2025-01-06 18:13:17,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:18,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:19,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:20,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:21,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:22,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:22,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:23,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:24,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:25,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:26,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:27,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:27,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:28,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:29,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:30,056 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:13:30,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:13:30,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:31,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:32,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:32,138 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:33,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:34,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:35,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:35,661 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:36,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:37,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:37,528 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:37,661 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:38,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:39,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:39,662 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:40,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:41,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:42,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:43,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:43,066 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:44,066 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:44,718 DEBUG SystemMonitor:2099720 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 18:13:44,718 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:13:45,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:13:45,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:13:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:46,066 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:47,066 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:48,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:48,152 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:49,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:50,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:51,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:52,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:53,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:54,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:54,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:55,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:55,666 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:13:56,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:57,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:58,067 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:13:59,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:13:59,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:00,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:14:00,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:14:00,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:01,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:02,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:03,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:04,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:04,193 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:05,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:06,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:07,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:08,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:09,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:10,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:10,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:11,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:12,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:13,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:13,294 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:14:13,294 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 with policy now +2025-01-06 18:14:13,295 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:14:13,296 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:14:13,297 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:14:13,299 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:14:13,523 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/t24i6gkl-media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 +2025-01-06 18:14:13,671 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:14:13,671 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 +2025-01-06 18:14:13,671 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:14:13,671 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos +2025-01-06 18:14:13,672 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:14:13,672 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media +2025-01-06 18:14:13,941 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/jnrwz3kr-media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 +2025-01-06 18:14:14,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:14,719 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:14:15,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:14:15,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:14:15,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:15,187 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:15,672 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:14:16,071 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:17,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:18,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:19,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:19,673 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:14:20,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:21,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:21,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:21,674 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:14:22,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:23,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:24,072 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:25,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:26,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:26,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:27,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:28,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:29,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:30,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:14:30,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:14:30,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:31,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:31,212 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:32,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:33,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:34,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:35,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:36,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:37,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:37,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:38,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:39,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:40,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:41,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:42,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:42,074 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:43,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:44,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:44,719 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:14:45,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:14:45,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:14:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:46,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:47,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:47,202 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:48,075 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:49,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:50,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:51,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:52,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:53,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:53,076 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:54,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:55,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:56,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:57,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:58,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:14:59,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:14:59,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:00,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:15:00,057 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:15:00,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:01,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:02,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:03,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:04,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:04,140 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:05,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:06,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:07,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:08,078 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:09,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:10,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:10,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:11,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:12,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:13,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:14,079 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:14,720 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:15:15,057 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:15:15,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:15:15,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:15,130 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:16,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:17,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:18,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:19,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:20,080 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:21,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:21,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:22,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:23,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:24,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:25,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:26,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:26,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:27,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:28,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:29,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:30,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:15:30,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:15:30,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:31,081 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:31,153 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:32,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:33,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:34,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:35,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:36,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:37,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:37,082 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:38,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:39,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:40,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:41,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:42,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:42,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:43,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:44,083 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:44,720 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:15:45,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:15:45,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:15:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:46,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:47,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:47,141 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:48,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:49,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:50,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:51,084 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:52,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:53,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:53,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:54,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:55,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:56,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:57,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:58,085 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:15:59,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:15:59,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:00,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:16:00,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:16:00,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:01,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:02,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:03,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:04,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:04,133 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:05,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:06,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:07,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:08,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:09,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:10,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:10,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:11,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:12,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:13,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:14,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:14,721 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:16:15,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:16:15,058 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:16:15,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:15,155 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:16,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:17,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:18,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:19,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:20,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:21,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:21,088 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:22,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:23,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:24,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:25,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:26,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:26,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:27,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:28,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:29,089 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:30,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:16:30,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:16:30,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:31,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:31,138 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:32,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:33,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:34,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:35,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:36,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:37,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:37,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:38,090 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:39,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:40,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:41,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:42,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:42,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:43,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:44,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:44,721 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:16:45,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:16:45,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:16:45,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:46,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:47,091 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:47,209 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:48,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:49,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:50,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:51,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:52,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:53,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:53,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:54,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:55,092 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:56,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:57,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:58,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:16:58,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:16:59,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:00,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:17:00,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:17:00,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:01,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:02,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:03,093 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:03,214 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:04,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:05,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:06,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:07,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:08,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:09,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:09,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:10,094 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:11,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:12,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:13,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:14,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:14,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:14,721 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:17:15,058 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:17:15,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:17:15,101 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:16,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:17,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:18,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:19,095 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:19,210 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:20,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:21,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:22,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:23,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:24,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:25,064 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:25,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:26,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:27,096 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:28,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:29,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:29,738 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:29,738 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:17:29,986 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/7mqcmgat-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:17:30,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:17:30,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:17:30,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:30,182 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:31,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:31,738 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:32,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:33,097 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:33,739 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:34,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:35,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:36,065 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:36,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:37,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:38,100 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:39,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:40,099 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:41,100 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:41,177 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:41,742 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:42,100 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:43,100 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:44,103 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:44,722 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:17:45,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:17:45,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:17:45,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:46,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:47,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:47,163 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:48,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:49,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:50,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:51,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:52,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:52,169 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:53,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:53,745 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:17:54,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:55,104 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:56,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:57,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:58,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:17:58,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:17:59,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:00,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:18:00,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:18:00,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:01,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:02,105 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:03,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:03,152 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:04,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:05,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:06,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:07,106 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:08,107 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:08,297 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:08,617 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:18:08,617 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 with policy now +2025-01-06 18:18:08,618 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:18:08,618 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:18:08,619 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:18:08,619 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:18:08,750 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:18:08,750 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 +2025-01-06 18:18:08,750 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:18:08,839 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/9bs6626v-media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 +2025-01-06 18:18:09,100 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/873d9gvr-media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 +2025-01-06 18:18:09,109 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:09,750 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:10,111 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:11,110 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:12,121 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:13,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:13,751 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:14,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:14,197 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:14,722 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:18:15,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:18:15,059 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:18:15,111 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:15,752 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:16,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:17,111 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:18,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:19,111 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:20,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:20,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:20,759 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:18:20,764 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:18:20,764 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:18:20,764 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:18:21,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:21,759 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:21,759 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:18:22,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:23,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:23,760 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:24,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:25,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:25,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:25,761 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:26,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:27,112 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:28,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:29,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:30,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:18:30,060 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:18:30,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:30,167 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:31,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:32,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:33,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:34,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:35,113 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:36,068 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:36,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:37,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:38,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:39,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:40,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:41,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:41,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:42,114 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:43,115 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:44,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:44,723 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:18:45,059 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:18:45,060 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:18:45,115 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:46,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:46,197 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:47,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:47,768 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:48,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:49,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:50,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:51,116 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:51,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:51,769 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:52,117 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:53,117 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:54,117 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:55,117 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:56,118 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:56,749 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:18:57,118 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:57,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:18:58,118 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:59,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:18:59,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:00,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:19:00,063 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:19:00,209 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:19:00,210 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:19:00,210 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 with policy now +2025-01-06 18:19:00,210 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:19:00,210 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:19:00,211 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:00,227 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:19:00,463 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/kkdndppd-media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 +2025-01-06 18:19:00,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:19:00,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 +2025-01-06 18:19:00,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:19:01,040 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/h7baghxu-media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 +2025-01-06 18:19:01,212 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:01,772 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:02,215 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:02,265 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:02,773 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:03,213 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:04,217 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:05,219 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:05,774 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:06,216 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:07,217 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:07,665 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:07,775 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:08,220 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:09,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:10,218 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:11,221 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:12,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:13,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:13,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:14,222 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:14,723 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:19:15,063 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:19:15,063 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:19:15,224 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:16,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:17,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:17,627 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:19:17,627 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:19:17,627 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:19:17,778 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:17,778 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:19:17,778 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:19:18,027 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/ecwvmssm-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:19:18,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:19,073 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:19,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:19,778 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:20,224 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:21,223 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:21,779 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:22,225 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:23,226 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:24,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:24,224 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:25,224 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:26,232 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:27,229 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:28,229 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:29,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:30,060 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:19:30,061 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:19:30,152 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:30,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:31,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:32,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:33,230 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:34,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:35,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:36,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:36,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:37,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:38,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:39,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:40,231 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:41,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:41,232 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:42,232 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:43,232 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:44,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:44,724 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:19:45,061 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:19:45,061 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:19:45,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:46,126 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:46,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:47,234 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:48,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:49,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:50,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:51,233 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:52,077 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:52,234 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:53,234 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:54,234 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:55,235 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:56,236 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:56,966 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:19:56,966 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 with policy now +2025-01-06 18:19:56,969 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:19:56,969 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:19:56,969 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:19:57,004 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:19:57,241 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:57,242 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/2g0e0a4w-media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 +2025-01-06 18:19:57,791 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:57,791 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:19:57,792 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 +2025-01-06 18:19:57,792 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:19:58,029 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:19:58,069 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/0ie7p8hp-media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 +2025-01-06 18:19:58,255 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:19:58,792 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:19:59,244 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:00,069 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:20:00,069 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:20:00,245 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:01,249 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:01,793 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:02,250 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:03,250 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:03,458 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:04,256 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:05,254 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:05,794 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:06,257 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:07,253 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:08,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:09,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:09,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:10,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:11,256 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:12,255 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:12,720 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:20:12,720 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:20:12,721 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:20:12,796 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:20:12,796 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:20:13,258 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:13,797 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:14,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:14,724 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:20:14,724 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:15,070 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:20:15,070 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:20:15,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:16,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:17,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:17,798 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:18,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:19,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:20,048 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:20,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:21,259 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:22,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:23,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:24,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:25,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:25,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:26,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:27,260 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:28,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:29,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:30,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:20:30,062 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:20:30,184 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:30,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:31,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:32,261 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:33,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:34,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:35,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:36,086 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:36,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:37,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:38,262 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:39,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:40,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:41,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:41,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:42,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:43,263 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:44,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:44,725 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:20:45,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:20:45,062 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:20:45,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:46,200 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:46,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:47,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:48,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:49,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:50,264 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:51,275 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:51,279 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:51,809 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:51,809 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 +2025-01-06 18:20:51,810 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval +2025-01-06 18:20:51,822 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:20:51,822 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 with policy now +2025-01-06 18:20:51,825 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:20:51,826 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:20:51,826 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:20:51,826 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:20:52,033 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/wrnj6d2q-media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 +2025-01-06 18:20:52,277 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:52,294 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/pqvst0yk-media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 +2025-01-06 18:20:52,810 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:20:53,282 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:53,810 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:54,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:54,810 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:55,289 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:56,283 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:56,521 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:20:57,301 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:57,811 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:58,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:59,294 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:20:59,803 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:20:59,803 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:20:59,803 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:20:59,811 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 18:20:59,811 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 18:20:59,812 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:20:59,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:20:59,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:00,016 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:21:00,052 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/5e1oia4n-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:00,062 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 18:21:00,286 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:00,345 INFO wandb-upload_1:2099720 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfjqaqtbv +2025-01-06 18:21:00,387 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 18:21:00,395 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 18:21:00,753 INFO SenderThread:2099720 [sender.py:send_request_log_artifact():1525] logged artifact run-kkp1x4wb-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjAwNDA0NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxOTYwMg==', 'latestArtifact': None}} +2025-01-06 18:21:00,753 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:21:00,753 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 18:21:00,754 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:21:00,754 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,755 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,755 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: stop_status +2025-01-06 18:21:00,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:00,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:00,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-06 18:21:00,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:21:00,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media +2025-01-06 18:21:00,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table +2025-01-06 18:21:00,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval +2025-01-06 18:21:00,853 DEBUG SenderThread:2099720 [sender.py:send():379] send: telemetry +2025-01-06 18:21:00,853 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,853 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,853 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,854 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,854 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,854 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,854 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,855 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,855 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,855 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,855 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,855 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,856 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,856 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,856 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,856 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,857 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,857 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,857 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,857 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,857 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:00,858 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:00,858 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 18:21:00,935 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/uwgcvwwj-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:21:01,036 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/ccuehux8-media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-06 18:21:01,188 INFO wandb-upload_0:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/vgccyunv-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:21:01,286 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:01,427 INFO wandb-upload_2:2099720 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5nz775i3 +2025-01-06 18:21:01,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:21:01,813 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:01,829 INFO SenderThread:2099720 [sender.py:send_request_log_artifact():1525] logged artifact run-kkp1x4wb-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjAwNDA3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxOTYwMw==', 'latestArtifact': None}} +2025-01-06 18:21:01,830 DEBUG SenderThread:2099720 [sender.py:send():379] send: files +2025-01-06 18:21:01,830 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json with policy now +2025-01-06 18:21:01,830 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:21:01,830 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:21:01,830 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:01,831 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:02,072 INFO wandb-upload_0:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/ih1ud28a-media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-06 18:21:02,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:02,814 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:03,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:04,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:05,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:06,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:07,087 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:21:07,287 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:07,717 DEBUG SenderThread:2099720 [sender.py:send():379] send: exit +2025-01-06 18:21:07,717 INFO SenderThread:2099720 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 18:21:07,717 INFO SenderThread:2099720 [sender.py:send_exit():588] handling runtime: 503 +2025-01-06 18:21:07,718 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:07,718 INFO SenderThread:2099720 [sender.py:send_exit():594] send defer +2025-01-06 18:21:07,718 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,718 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 18:21:07,718 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:07,718 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 18:21:07,718 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 1 +2025-01-06 18:21:07,718 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,718 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 18:21:07,719 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:07,719 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 18:21:07,719 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 2 +2025-01-06 18:21:07,719 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,719 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 18:21:07,719 INFO HandlerThread:2099720 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 18:21:07,719 DEBUG SystemMonitor:2099720 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 18:21:07,719 DEBUG SystemMonitor:2099720 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 18:21:07,719 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 18:21:07,720 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 18:21:07,771 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 18:21:07,772 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 18:21:07,772 INFO HandlerThread:2099720 [interfaces.py:finish():200] Joined network monitor +2025-01-06 18:21:07,772 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:07,772 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 18:21:07,772 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 3 +2025-01-06 18:21:07,772 DEBUG SenderThread:2099720 [sender.py:send():379] send: stats +2025-01-06 18:21:07,773 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,773 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 18:21:07,773 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:07,773 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 18:21:07,773 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 4 +2025-01-06 18:21:07,773 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:07,773 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 18:21:07,816 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:09,817 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:21:13,808 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:13,808 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:21:13,808 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 18:21:13,810 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:13,810 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 18:21:13,810 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 5 +2025-01-06 18:21:13,810 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:13,810 DEBUG SenderThread:2099720 [sender.py:send():379] send: history +2025-01-06 18:21:13,810 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: summary_record +2025-01-06 18:21:13,810 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:13,810 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 18:21:13,811 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:13,811 DEBUG SenderThread:2099720 [sender.py:send():379] send: summary +2025-01-06 18:21:13,811 INFO SenderThread:2099720 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 18:21:13,812 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:13,812 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 18:21:13,812 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 6 +2025-01-06 18:21:13,812 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:13,812 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 18:21:13,812 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:13,812 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 18:21:13,815 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 18:21:13,818 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:13,952 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 7 +2025-01-06 18:21:13,952 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:13,952 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 18:21:13,952 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:13,952 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 18:21:14,718 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:14,819 INFO Thread-12 :2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml +2025-01-06 18:21:16,337 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 8 +2025-01-06 18:21:16,337 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:16,337 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:16,337 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 18:21:16,338 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:16,338 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 18:21:16,338 INFO SenderThread:2099720 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 18:21:16,340 INFO SenderThread:2099720 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 18:21:16,342 INFO SenderThread:2099720 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 18:21:16,355 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 9 +2025-01-06 18:21:16,356 DEBUG SenderThread:2099720 [sender.py:send():379] send: artifact +2025-01-06 18:21:16,356 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:16,359 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 18:21:16,572 INFO SenderThread:2099720 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 18:21:16,572 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:16,572 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 18:21:16,572 INFO SenderThread:2099720 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 18:21:16,719 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:16,820 INFO SenderThread:2099720 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:21:16,820 INFO SenderThread:2099720 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files +2025-01-06 18:21:16,820 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml config.yaml +2025-01-06 18:21:16,821 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json wandb-summary.json +2025-01-06 18:21:16,821 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/requirements.txt requirements.txt +2025-01-06 18:21:16,823 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-metadata.json wandb-metadata.json +2025-01-06 18:21:16,823 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:16,823 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log output.log +2025-01-06 18:21:16,823 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/diff.patch diff.patch +2025-01-06 18:21:16,826 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 media/videos/eval/0_eval_0_9467d3a3a29eb4dd688a.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 media/videos/eval/4_eval_4_3009e51676e13f8bfc1c.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 media/videos/eval/2_eval_2_426b092ff928a3231ed1.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 media/videos/eval/3_eval_3_6b5ff732fe53abaf1668.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 media/videos/eval/1_eval_1_6baff3ebaf384ea295b4.mp4 +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 18:21:16,827 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 10 +2025-01-06 18:21:16,827 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:16,829 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:16,830 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 18:21:16,830 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:16,830 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 18:21:16,831 INFO SenderThread:2099720 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 18:21:16,996 INFO wandb-upload_1:2099720 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/requirements.txt +2025-01-06 18:21:17,035 INFO wandb-upload_0:2099720 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/wandb-summary.json +2025-01-06 18:21:17,049 INFO wandb-upload_2:2099720 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/config.yaml +2025-01-06 18:21:17,101 INFO wandb-upload_4:2099720 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/files/output.log +2025-01-06 18:21:17,111 INFO wandb-upload_3:2099720 [upload_job.py:push():130] Uploaded file /tmp/tmp5lkhljsjwandb/78bt69nu-events.out.tfevents.1736205165.leonmkim-ROG-Strix-G15CS-G15CS.2099546.0 +2025-01-06 18:21:17,311 INFO Thread-11 (_thread_body):2099720 [sender.py:transition_state():614] send defer: 11 +2025-01-06 18:21:17,312 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:17,312 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 18:21:17,312 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:17,312 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 18:21:17,312 INFO SenderThread:2099720 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 18:21:17,313 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 12 +2025-01-06 18:21:17,313 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:17,313 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 18:21:17,314 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:17,314 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 18:21:17,314 INFO SenderThread:2099720 [file_stream.py:finish():601] file stream finish called +2025-01-06 18:21:17,390 INFO SenderThread:2099720 [file_stream.py:finish():605] file stream finish is done +2025-01-06 18:21:17,390 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 13 +2025-01-06 18:21:17,390 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:17,390 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 18:21:17,390 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:17,390 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 18:21:17,390 INFO SenderThread:2099720 [sender.py:transition_state():614] send defer: 14 +2025-01-06 18:21:17,390 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: defer +2025-01-06 18:21:17,390 INFO HandlerThread:2099720 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 18:21:17,391 DEBUG SenderThread:2099720 [sender.py:send():379] send: final +2025-01-06 18:21:17,391 DEBUG SenderThread:2099720 [sender.py:send():379] send: footer +2025-01-06 18:21:17,391 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: defer +2025-01-06 18:21:17,391 INFO SenderThread:2099720 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 18:21:17,391 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:17,392 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 18:21:17,392 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:17,392 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 18:21:17,392 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: server_info +2025-01-06 18:21:17,392 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 18:21:17,394 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 18:21:17,395 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 18:21:17,458 DEBUG SenderThread:2099720 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 18:21:17,458 INFO MainThread:2099720 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 18:21:17,458 INFO MainThread:2099720 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 18:21:17,459 INFO MainThread:2099720 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 18:21:17,459 DEBUG HandlerThread:2099720 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 18:21:17,459 INFO HandlerThread:2099720 [handler.py:finish():882] shutting down handler +2025-01-06 18:21:18,395 INFO WriterThread:2099720 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/run-kkp1x4wb.wandb +2025-01-06 18:21:18,458 INFO SenderThread:2099720 [sender.py:finish():1615] shutting down sender +2025-01-06 18:21:18,458 INFO SenderThread:2099720 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 18:21:18,458 INFO SenderThread:2099720 [file_pusher.py:join():175] waiting for file pusher diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug.log b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6216f865ffab169fe8aebdcfbe5bc7ef35246b0f --- /dev/null +++ b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 18:12:44,347 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Configure stats pid to 2099546 +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/settings +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug.log +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/logs/debug-internal.log +2025-01-06 18:12:44,348 INFO MainThread:2099546 [wandb_init.py:init():569] calling init triggers +2025-01-06 18:12:44,349 INFO MainThread:2099546 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb'} +2025-01-06 18:12:44,349 INFO MainThread:2099546 [wandb_init.py:init():619] starting backend +2025-01-06 18:12:44,349 INFO MainThread:2099546 [wandb_init.py:init():623] setting up manager +2025-01-06 18:12:44,352 INFO MainThread:2099546 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 18:12:44,355 INFO MainThread:2099546 [wandb_init.py:init():631] backend started and connected +2025-01-06 18:12:44,366 INFO MainThread:2099546 [wandb_init.py:init():720] updated telemetry +2025-01-06 18:12:44,374 INFO MainThread:2099546 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 18:12:44,663 INFO MainThread:2099546 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 18:12:44,703 INFO MainThread:2099546 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 18:12:44,703 INFO MainThread:2099546 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 18:12:45,057 INFO MainThread:2099546 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 18:12:45,057 INFO MainThread:2099546 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 18:12:45,057 INFO MainThread:2099546 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 18:12:45,057 INFO MainThread:2099546 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 18:12:45,058 INFO MainThread:2099546 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 18:12:45,058 INFO MainThread:2099546 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/kkp1x4wb/tb, True +2025-01-06 18:12:53,040 INFO MainThread:2099546 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 18:21:18,460 WARNING MsgRouterThr:2099546 [router.py:message_loop():77] message_loop has been closed diff --git a/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/run-kkp1x4wb.wandb b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/run-kkp1x4wb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..342b1c978e57767819ebb9d9c3d51d1fb92efab1 Binary files /dev/null and b/kkp1x4wb/wandb/run-20250106_181244-kkp1x4wb/run-kkp1x4wb.wandb differ diff --git a/s3ldfxu8/.hydra/config.yaml b/s3ldfxu8/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/s3ldfxu8/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/s3ldfxu8/.hydra/hydra.yaml b/s3ldfxu8/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b00b64e23d489c11416f9f7ee94d7957536a093f --- /dev/null +++ b/s3ldfxu8/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/s3ldfxu8 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/s3ldfxu8/.hydra/overrides.yaml b/s3ldfxu8/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/s3ldfxu8/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/s3ldfxu8/eval_policy.log b/s3ldfxu8/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..816cc62ef5fd6d34e3475cdbee3d58177ae19be8 --- /dev/null +++ b/s3ldfxu8/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 19:57:23,863][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 19:57:23,866][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 19:57:27,051][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/snapshot_11249.pt b/snapshot_11249.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c724dd607b33eacd15026ed0ed483e38bf25554 --- /dev/null +++ b/snapshot_11249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be077b0f9c0c7071943385aefd2a49ad2b0dcf72bca25f4a64040de801b5f437 +size 910445958 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..d88f19c32dd90342a65ff7e7ec067486310108ab --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ab269fd7e9e7977039b59ad5ebd837a37d772e32811c61f6c68c255ae73c605 +size 910445958 diff --git a/snapshot_12937.pt b/snapshot_12937.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bccad61b38dd6a42f9643f81e0ad6721136f565 --- /dev/null +++ b/snapshot_12937.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a21852d8a8ab56dd4c11c6e2c395175145ee48b8e7466fd52c87152144cda84b +size 910445958 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..1de4400e2118aab19548603b8abf2096bc3e46e1 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24c1862cb4e25d19a53840b31c8acf83673453d4c45dd4190fef23e797c05f37 +size 910445958 diff --git a/snapshot_13999.pt b/snapshot_13999.pt new file mode 100644 index 0000000000000000000000000000000000000000..d52cb855bd148083019602eb236fc02afa57f127 --- /dev/null +++ b/snapshot_13999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af820cce2535f0d975fe122361033890abd1eba2685f2a4ca4bc23dc794fabe5 +size 910445958 diff --git a/snapshot_14465.pt b/snapshot_14465.pt new file mode 100644 index 0000000000000000000000000000000000000000..625bc4e4f059dee4448d6bb4700a2c70decfd0b2 --- /dev/null +++ b/snapshot_14465.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f78b455c6779262f75d4dce91c47228ea772b23b056b4d108f6a3cb89e9d47a3 +size 910445958 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..756d622e9b8eaa2e2320209756dc318f86ca991d --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a280cf64a617663ab8dc10fa3ddfd535bd50ed7ca4d4b4d15ec00e5faff4107 +size 910445958 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..67f4fce6e8afb101ad3f305373feddf460df39e5 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b4d90aaddcbc582a533bca77cbfe7646a7ae53b0d3e99b6f363f66fc3b3d4ec +size 910445958 diff --git a/snapshot_5999.pt b/snapshot_5999.pt new file mode 100644 index 0000000000000000000000000000000000000000..9004aeda1ab818b39330f207328ffd9eaba09ff9 --- /dev/null +++ b/snapshot_5999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2df003e362f3417722c205510adaf78ed442cfe8cb0124b98ea20db368a448d9 +size 910445958 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..27b18038c0db329d04429f16792cfedcbe7e0144 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d168705f1cdf4b0f090a26d7841e1b03b73ed59d407d507aee0c07e77648ea6 +size 910445958 diff --git a/snapshot_7044.pt b/snapshot_7044.pt new file mode 100644 index 0000000000000000000000000000000000000000..16049a330c9042a2fc509c24db12c150c9d247ec --- /dev/null +++ b/snapshot_7044.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0b1619b2a957f84799b8e05e432c0445949f1bfe03e85e9c1568802c8298561 +size 910445958 diff --git a/snapshot_7249.pt b/snapshot_7249.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f5c9eabf24c103901900d6db1eceff10be80fb3 --- /dev/null +++ b/snapshot_7249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:574b4d76880dcec84506a67f6cc663e286ff1a0ff6941a03dc212a4d387b44bd +size 910445958 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..64fa7e9ea04d6733e0d22785ca85267fd44dfd5d --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce1f2f8993056769272bdfdc6281108e2adfcc19f79ee8e41876c83aa206f5f2 +size 910445958 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..98904db2cc40079492f09ab10fe8b8172bb9cfd7 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f99ad395b2c76eb3711373821c45f1456dc6282924a65b1ddf0e7b163193777 +size 910445958 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..756d622e9b8eaa2e2320209756dc318f86ca991d --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a280cf64a617663ab8dc10fa3ddfd535bd50ed7ca4d4b4d15ec00e5faff4107 +size 910445958 diff --git a/urh96fgp/.hydra/config.yaml b/urh96fgp/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/urh96fgp/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/urh96fgp/.hydra/hydra.yaml b/urh96fgp/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3da20d1d555426eb510644dfac622b540f297cc5 --- /dev/null +++ b/urh96fgp/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/urh96fgp/.hydra/overrides.yaml b/urh96fgp/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/urh96fgp/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/urh96fgp/episode_rosbags/aligned_depth_to_color_K.npy b/urh96fgp/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/urh96fgp/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/urh96fgp/episode_rosbags/cam_tf_world.npy b/urh96fgp/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/urh96fgp/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/urh96fgp/episode_rosbags/color_K.npy b/urh96fgp/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/urh96fgp/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/urh96fgp/episode_rosbags/depth_K.npy b/urh96fgp/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/urh96fgp/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/urh96fgp/episode_rosbags/episode_2_2025-01-07-20-13-17.bag b/urh96fgp/episode_rosbags/episode_2_2025-01-07-20-13-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..37c70b1bc6a3f8a76bfdbd33f9c7e19192dd9b4b --- /dev/null +++ b/urh96fgp/episode_rosbags/episode_2_2025-01-07-20-13-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75af94d41cdc66bae899d68bded3c4154f37ba2934251d788417fff537acea62 +size 1059689553 diff --git a/urh96fgp/eval_robot.log b/urh96fgp/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..596e0de1faaee52e7d2fac3832d07e74935c88e3 --- /dev/null +++ b/urh96fgp/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 20:11:14,682][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:11:14,685][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/urh96fgp/eval_video/0_eval.mp4 b/urh96fgp/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d2876abfdde5c474f62ed811ccee43932850d485 --- /dev/null +++ b/urh96fgp/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4ea3df26327a48d1e752e90490983d710c64c6eb48ced1b5717e657466563f0 +size 1163698 diff --git a/urh96fgp/eval_video/1_eval.mp4 b/urh96fgp/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf9d7e047ea818c7d12c79dcac1d3e5aa979329b --- /dev/null +++ b/urh96fgp/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4296e644b36d9e3925a9a864e3e6dcd3aad334baf0d748dbf4c697fa10575ff2 +size 1096196 diff --git a/urh96fgp/eval_video/2_eval.mp4 b/urh96fgp/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45e0d6bfbee4547db4c920dfc6d8502c1cf671ad --- /dev/null +++ b/urh96fgp/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09f9d609000e258b620335c623ae25f739a95ae910ce21f6a01bbb2442d21fa5 +size 680759 diff --git a/urh96fgp/eval_video/3_eval.mp4 b/urh96fgp/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e2545aec4571fe8b7ef8324afee81f76a814223 --- /dev/null +++ b/urh96fgp/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b814eef54727e0c0f4acb8be2cd0ad22d83cb7c3153b4deb14d42fe1cee7428 +size 1364301 diff --git a/urh96fgp/eval_video/4_eval.mp4 b/urh96fgp/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aacb89ee5c4aff8031b39e85568daa941104c764 --- /dev/null +++ b/urh96fgp/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff0c295868f146e8406664ee1dbe5f8d174a53efdbc91e2590c0ecbebe7e64c6 +size 1329769 diff --git a/urh96fgp/tb/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 b/urh96fgp/tb/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 new file mode 100644 index 0000000000000000000000000000000000000000..b453b70a7ba816eb2b9c240e815335cd9531ee7a --- /dev/null +++ b/urh96fgp/tb/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c9b451ce48b19436504fd40f73cbcbb0e84dec7010e44f72a473e38eb3fb2bc +size 6519 diff --git a/urh96fgp/wandb/debug-internal.log b/urh96fgp/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bd5496c5da5b7e43b8d494d79da2322044f03e4b --- /dev/null +++ b/urh96fgp/wandb/debug-internal.log @@ -0,0 +1,756 @@ +2025-01-07 20:11:20,661 INFO StreamThr :2221501 [internal.py:wandb_internal():85] W&B internal server running at pid: 2221501, started at: 2025-01-07 20:11:20.660615 +2025-01-07 20:11:20,661 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:11:20,675 INFO WriterThread:2221501 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/run-urh96fgp.wandb +2025-01-07 20:11:20,675 DEBUG SenderThread:2221501 [sender.py:send():379] send: header +2025-01-07 20:11:20,688 DEBUG SenderThread:2221501 [sender.py:send():379] send: run +2025-01-07 20:11:20,894 INFO SenderThread:2221501 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files +2025-01-07 20:11:20,894 INFO SenderThread:2221501 [sender.py:_start_run_threads():1188] run started: urh96fgp with start time 1736298680.662175 +2025-01-07 20:11:20,911 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:11:20,912 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:11:20,955 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:11:20,960 DEBUG HandlerThread:2221501 [system_info.py:__init__():26] System info init +2025-01-07 20:11:20,960 DEBUG HandlerThread:2221501 [system_info.py:__init__():41] System info init done +2025-01-07 20:11:20,960 INFO HandlerThread:2221501 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:11:20,960 INFO SystemMonitor:2221501 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:11:20,960 INFO HandlerThread:2221501 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:11:20,961 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:11:20,961 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:11:20,962 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:11:20,962 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:11:20,963 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:11:20,973 DEBUG HandlerThread:2221501 [system_info.py:probe():152] Probing system +2025-01-07 20:11:20,974 DEBUG HandlerThread:2221501 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:11:20,979 DEBUG HandlerThread:2221501 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:11:20,979 DEBUG HandlerThread:2221501 [system_info.py:probe():200] Probing system done +2025-01-07 20:11:20,979 DEBUG HandlerThread:2221501 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:11:20.973511', 'startedAt': '2025-01-08T01:11:20.653238', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.282, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4466.498, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.02, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.992, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 554.1286163330078}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:11:20,979 INFO HandlerThread:2221501 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:11:20,979 INFO HandlerThread:2221501 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:11:20,979 DEBUG HandlerThread:2221501 [system_info.py:_save_code():44] Saving code +2025-01-07 20:11:20,981 DEBUG HandlerThread:2221501 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:11:20,981 DEBUG HandlerThread:2221501 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:11:21,010 DEBUG HandlerThread:2221501 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:11:21,010 INFO HandlerThread:2221501 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:11:21,014 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:11:21,014 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:11:21,014 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:11:21,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:11:21,267 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:11:21,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:21,269 DEBUG SenderThread:2221501 [sender.py:send():379] send: telemetry +2025-01-07 20:11:21,289 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/rxji41gk-wandb-metadata.json +2025-01-07 20:11:21,293 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/f4kmqkjd-code/FISH/eval_robot.py +2025-01-07 20:11:21,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:11:21,309 INFO HandlerThread:2221501 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/tb" + save: true +} +control { + relay_id: "b30f97c685eb4f2885a4012e0da1b4ce" +} +_info { + stream_id: "urh96fgp" +} + +2025-01-07 20:11:21,309 DEBUG HandlerThread:2221501 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:11:21,310 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:11:21,448 DEBUG SenderThread:2221501 [sender.py:send():379] send: tbrecord +2025-01-07 20:11:21,449 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:11:21,449 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 with policy live +2025-01-07 20:11:21,641 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/d8m4ti26-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:11:21,896 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/requirements.txt +2025-01-07 20:11:21,896 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:11:21,896 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:21,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code/FISH/eval_robot.py +2025-01-07 20:11:21,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-metadata.json +2025-01-07 20:11:21,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code/FISH +2025-01-07 20:11:21,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code +2025-01-07 20:11:22,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:23,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:23,895 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:24,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:25,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:25,533 DEBUG SenderThread:2221501 [sender.py:send():379] send: config +2025-01-07 20:11:26,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:26,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:27,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:27,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:28,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:29,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:30,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:31,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:31,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:31,898 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:32,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:33,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:34,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:35,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:35,899 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:36,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:11:36,267 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:11:36,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:36,330 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:37,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:38,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:39,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:39,900 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:40,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:41,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:41,494 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:42,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:43,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:43,901 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:44,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:45,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:46,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:47,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:47,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:48,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:49,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:50,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:51,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:11:51,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:11:51,313 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:52,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:52,406 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:52,903 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml +2025-01-07 20:11:53,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:54,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:55,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:56,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:57,274 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:58,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:58,274 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:59,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:59,905 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:00,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:01,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:02,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:03,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:04,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:04,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:05,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:06,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:12:06,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:12:06,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:07,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:08,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:08,955 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:12:08,955 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:12:08,955 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 with policy now +2025-01-07 20:12:08,956 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:12:08,956 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:12:08,958 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:12:09,206 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/tqh7i2v8-media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 +2025-01-07 20:12:09,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos +2025-01-07 20:12:09,967 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:10,213 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/gyo0vgzq-media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 +2025-01-07 20:12:10,278 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:11,278 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:12,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:13,278 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:13,910 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:14,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:15,185 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:15,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:16,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:17,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:17,911 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:18,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:19,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:20,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:20,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:20,963 DEBUG SystemMonitor:2221501 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:12:20,963 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:12:21,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:12:21,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:12:21,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:22,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:23,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:24,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:25,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:25,352 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:26,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:27,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:28,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:29,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:29,915 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:12:30,177 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/06kf33l8-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:12:30,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:30,465 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:31,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:31,916 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:32,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:33,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:34,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:35,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:35,466 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:35,917 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:36,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:12:36,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:12:36,313 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:37,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:38,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:39,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:40,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:41,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:41,282 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:42,282 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:43,282 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:43,920 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:44,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:45,283 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:46,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:46,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:47,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:48,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:49,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:50,285 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:50,964 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:12:51,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:12:51,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:12:51,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:51,415 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:52,285 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:53,285 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:53,923 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:54,285 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:55,293 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:56,293 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:57,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:57,293 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:58,293 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:59,294 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:00,297 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:00,457 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:13:00,457 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 with policy now +2025-01-07 20:13:00,460 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:13:00,461 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:13:00,461 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:13:00,461 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:13:00,699 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/hz4ko86e-media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 +2025-01-07 20:13:00,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:13:00,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 +2025-01-07 20:13:00,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:13:01,198 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/m3177g2j-media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 +2025-01-07 20:13:01,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:01,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:02,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:02,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:02,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:03,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:04,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:05,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:05,926 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:06,277 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:13:06,277 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:13:06,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:07,298 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:07,927 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:08,005 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:08,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:09,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:10,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:11,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:12,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:13,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:13,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:13,929 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:13:14,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:14,716 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:13:14,716 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:13:14,716 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:13:14,929 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:13:15,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:15,930 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:16,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:17,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:18,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:18,697 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:19,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:19,931 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:20,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:20,965 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:13:21,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:13:21,269 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:13:21,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:22,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:23,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:24,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:24,357 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:25,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:26,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:27,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:28,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:29,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:30,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:30,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:31,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:32,305 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:33,278 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:13:33,278 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 with policy now +2025-01-07 20:13:33,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:13:33,281 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:13:33,281 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:13:33,282 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:13:33,307 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:33,553 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/ur6qagdb-media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 +2025-01-07 20:13:33,935 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:13:33,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:33,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 +2025-01-07 20:13:33,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:13:34,197 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/xh7y27ox-media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 +2025-01-07 20:13:34,310 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:34,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:35,305 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:35,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:35,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:36,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:13:36,276 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:13:36,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:37,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:37,937 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:38,319 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:39,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:40,315 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:40,761 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:41,315 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:41,938 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:42,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:43,314 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:44,314 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:45,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:46,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:46,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:47,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:48,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:49,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:50,315 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:50,539 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:13:50,539 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:13:50,550 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:13:50,941 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:13:50,942 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:13:50,965 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:13:51,224 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/uev3j49l-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:13:51,305 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:13:51,305 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:51,306 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:13:51,325 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:51,942 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:52,322 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:53,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:53,942 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:54,315 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:55,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:55,943 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:56,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:57,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:57,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:58,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:59,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:00,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:01,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:01,945 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:02,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:03,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:03,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:04,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:05,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:06,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:14:06,270 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:14:06,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:07,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:08,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:08,330 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:09,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:10,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:11,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:12,319 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:13,319 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:14,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:14,319 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:15,320 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:16,320 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:17,320 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:18,320 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:19,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:19,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:20,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:20,965 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:14:21,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:14:21,270 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:14:21,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:22,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:23,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:24,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:24,364 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:25,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:26,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:27,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:28,324 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:29,333 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:29,568 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:29,955 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:30,076 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:14:30,076 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 with policy now +2025-01-07 20:14:30,082 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:14:30,083 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:14:30,083 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:14:30,088 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:14:30,328 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:30,347 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/m8z85ohs-media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 +2025-01-07 20:14:30,956 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:14:30,956 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 +2025-01-07 20:14:30,956 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:14:31,245 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/zl5hyo3l-media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 +2025-01-07 20:14:31,332 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:31,956 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:32,326 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:33,331 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:34,331 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:34,689 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:35,332 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:35,957 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:36,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:14:36,284 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:14:36,333 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:37,334 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:37,958 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:38,340 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:39,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:40,336 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:40,434 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:41,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:42,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:43,337 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:43,603 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:14:43,603 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:14:43,604 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:14:43,960 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:14:43,960 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:43,960 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:14:44,339 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:45,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:45,961 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:46,157 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:46,347 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:47,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:47,961 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:48,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:49,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:50,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:50,966 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:14:51,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:14:51,271 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:14:51,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:51,385 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:52,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:53,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:54,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:55,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:55,964 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:56,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:57,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:57,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:58,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:59,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:00,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:01,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:02,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:02,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:03,344 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:04,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:05,344 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:06,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:15:06,271 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:15:06,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:07,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:07,420 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:08,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:09,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:10,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:11,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:12,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:13,277 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:13,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:14,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:15,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:16,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:17,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:18,347 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:18,842 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:19,351 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:19,972 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:20,224 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:15:20,224 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 with policy now +2025-01-07 20:15:20,228 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:15:20,228 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:20,229 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:20,229 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:20,349 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:20,500 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/dubhrrmg-media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 +2025-01-07 20:15:20,972 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:20,973 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 +2025-01-07 20:15:20,973 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:15:21,006 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:15:21,291 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:15:21,291 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:15:21,297 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/mnpvnpde-media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 +2025-01-07 20:15:21,355 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:21,973 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:22,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:22,973 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:23,357 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:23,977 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:24,356 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:25,358 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:25,974 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:26,364 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:27,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:27,975 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:28,375 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:29,362 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:29,837 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:30,365 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:31,364 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:31,628 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:15:31,628 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:15:31,803 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:15:31,976 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:31,976 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:31,976 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:15:31,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media +2025-01-07 20:15:31,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table +2025-01-07 20:15:31,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval +2025-01-07 20:15:32,155 INFO wandb-upload_1:2221501 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpezyg0hgw +2025-01-07 20:15:32,188 INFO wandb-upload_2:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/bkk4uymf-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:15:32,194 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:15:32,204 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:15:32,261 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/sp33j6e1-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:32,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:32,551 INFO SenderThread:2221501 [sender.py:send_request_log_artifact():1525] logged artifact run-urh96fgp-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM2NjY4NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyNjY5Nw==', 'latestArtifact': None}} +2025-01-07 20:15:32,551 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:32,551 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,551 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,552 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:15:32,552 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:15:32,552 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:32,552 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,555 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,556 DEBUG SenderThread:2221501 [sender.py:send():379] send: telemetry +2025-01-07 20:15:32,556 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,556 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,556 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,557 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,557 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,557 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,557 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,557 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,558 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,558 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,558 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,558 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,558 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,559 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,559 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,559 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,559 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,560 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,560 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,560 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,560 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,561 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,561 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:15:32,806 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/ybong3xl-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:15:32,952 INFO wandb-upload_0:2221501 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1yq2keh3 +2025-01-07 20:15:32,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:32,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:32,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json +2025-01-07 20:15:32,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval +2025-01-07 20:15:33,318 INFO SenderThread:2221501 [sender.py:send_request_log_artifact():1525] logged artifact run-urh96fgp-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM2NjcyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyNjcwMg==', 'latestArtifact': None}} +2025-01-07 20:15:33,320 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:15:33,320 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json with policy now +2025-01-07 20:15:33,321 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:33,321 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:33,322 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:33,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:33,523 INFO wandb-upload_2:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/yg19ecs8-media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json +2025-01-07 20:15:33,772 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/wcxjbygd-media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json +2025-01-07 20:15:33,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:33,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:34,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:35,322 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:35,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:36,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:15:36,272 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:15:36,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:37,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:38,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:39,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:40,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:40,410 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:40,980 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml +2025-01-07 20:15:41,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:42,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:43,285 DEBUG SenderThread:2221501 [sender.py:send():379] send: exit +2025-01-07 20:15:43,285 INFO SenderThread:2221501 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:15:43,285 INFO SenderThread:2221501 [sender.py:send_exit():588] handling runtime: 262 +2025-01-07 20:15:43,286 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:send_exit():594] send defer +2025-01-07 20:15:43,287 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,287 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:15:43,287 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:15:43,287 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,287 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:15:43,287 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:15:43,287 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,287 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:15:43,288 INFO HandlerThread:2221501 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:15:43,288 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:15:43,288 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:15:43,288 DEBUG SystemMonitor:2221501 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:15:43,288 DEBUG SystemMonitor:2221501 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:15:43,341 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:15:43,342 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:15:43,342 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:15:43,342 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:43,342 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:15:43,342 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:15:43,342 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:15:43,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,343 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:15:43,343 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:43,343 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:15:43,343 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:15:43,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,343 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:15:43,981 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:43,981 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:49,663 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:49,664 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:49,664 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:15:49,665 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:49,665 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:15:49,665 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:15:49,665 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:49,665 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:49,666 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:49,666 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:49,666 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:15:49,667 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:49,667 DEBUG SenderThread:2221501 [sender.py:send():379] send: summary +2025-01-07 20:15:49,667 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:49,667 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:49,667 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:15:49,668 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:15:49,668 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:49,668 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:15:49,668 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:49,668 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:15:49,668 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:15:49,668 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:49,668 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:49,668 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:15:49,668 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:49,669 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:15:49,984 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:50,286 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:50,457 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:15:50,457 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:50,457 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:50,457 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:15:50,458 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:50,458 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:15:50,458 INFO SenderThread:2221501 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:15:50,460 INFO SenderThread:2221501 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:15:50,462 INFO SenderThread:2221501 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:15:50,475 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:15:50,475 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:50,475 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:15:50,476 DEBUG SenderThread:2221501 [sender.py:send():379] send: artifact +2025-01-07 20:15:50,781 INFO SenderThread:2221501 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:15:50,782 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:50,782 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:15:50,782 INFO SenderThread:2221501 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:15:50,984 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:50,985 INFO SenderThread:2221501 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files +2025-01-07 20:15:50,985 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml config.yaml +2025-01-07 20:15:50,985 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json wandb-summary.json +2025-01-07 20:15:50,985 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:50,988 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/requirements.txt requirements.txt +2025-01-07 20:15:50,988 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:15:50,988 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log output.log +2025-01-07 20:15:50,988 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 +2025-01-07 20:15:50,993 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json +2025-01-07 20:15:50,993 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:15:50,993 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:15:50,993 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:50,993 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:15:50,994 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:50,994 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:15:50,994 INFO SenderThread:2221501 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:15:51,160 INFO wandb-upload_2:2221501 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml +2025-01-07 20:15:51,164 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:51,210 INFO wandb-upload_3:2221501 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/requirements.txt +2025-01-07 20:15:51,239 INFO wandb-upload_4:2221501 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:51,243 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/kcrplpt4-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:51,287 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:51,287 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:51,443 INFO Thread-11 (_thread_body):2221501 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:15:51,443 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:51,443 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:15:51,443 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:51,443 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:15:51,443 INFO SenderThread:2221501 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:15:51,445 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:15:51,445 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:51,445 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:15:51,445 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:51,445 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:15:51,445 INFO SenderThread:2221501 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:15:51,674 INFO SenderThread:2221501 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:15:51,674 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:15:51,674 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:51,674 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:15:51,674 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:51,674 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:15:51,674 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:15:51,674 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:51,674 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:15:51,675 DEBUG SenderThread:2221501 [sender.py:send():379] send: final +2025-01-07 20:15:51,675 DEBUG SenderThread:2221501 [sender.py:send():379] send: footer +2025-01-07 20:15:51,675 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:51,675 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:15:51,675 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:51,675 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:15:51,676 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:51,676 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:15:51,676 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:51,676 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:15:51,677 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:15:51,677 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:51,737 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:51,737 INFO MainThread:2221501 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:15:51,738 INFO MainThread:2221501 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:15:51,738 INFO MainThread:2221501 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:15:51,738 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:15:51,738 INFO HandlerThread:2221501 [handler.py:finish():882] shutting down handler +2025-01-07 20:15:52,676 INFO WriterThread:2221501 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/run-urh96fgp.wandb +2025-01-07 20:15:52,737 INFO SenderThread:2221501 [sender.py:finish():1615] shutting down sender +2025-01-07 20:15:52,737 INFO SenderThread:2221501 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:15:52,737 INFO SenderThread:2221501 [file_pusher.py:join():175] waiting for file pusher diff --git a/urh96fgp/wandb/debug.log b/urh96fgp/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d96ce063879470bc557fe430ae878acb2c761282 --- /dev/null +++ b/urh96fgp/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Configure stats pid to 2221061 +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/settings +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug.log +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug-internal.log +2025-01-07 20:11:20,656 INFO MainThread:2221061 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:11:20,656 INFO MainThread:2221061 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp'} +2025-01-07 20:11:20,656 INFO MainThread:2221061 [wandb_init.py:init():619] starting backend +2025-01-07 20:11:20,656 INFO MainThread:2221061 [wandb_init.py:init():623] setting up manager +2025-01-07 20:11:20,659 INFO MainThread:2221061 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:11:20,661 INFO MainThread:2221061 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:11:20,675 INFO MainThread:2221061 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:11:20,682 INFO MainThread:2221061 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:11:20,911 INFO MainThread:2221061 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:11:20,949 INFO MainThread:2221061 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:11:20,950 INFO MainThread:2221061 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:11:21,267 INFO MainThread:2221061 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:11:21,268 INFO MainThread:2221061 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:11:21,268 INFO MainThread:2221061 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:11:21,268 INFO MainThread:2221061 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:11:21,270 INFO MainThread:2221061 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:11:21,270 INFO MainThread:2221061 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/tb, True +2025-01-07 20:11:25,532 INFO MainThread:2221061 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:15:52,739 WARNING MsgRouterThr:2221061 [router.py:message_loop():77] message_loop has been closed diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code/FISH/eval_robot.py b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4aae790ec85fec8c86a2c933b6cb039db0935e59 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736298680 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6fd31af5fb2517d379753e98e981911247129ff4 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[37, 60, 0.6166666666666667, 0.46349006891980815, 0.7536167203872801, 0.4929127618684639, 0.7386003359675044, 0.5128696039293299, 0.7125004790298323, 0.5289849340857711, 0.6996142845201615]]} \ No newline at end of file diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d2876abfdde5c474f62ed811ccee43932850d485 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4ea3df26327a48d1e752e90490983d710c64c6eb48ced1b5717e657466563f0 +size 1163698 diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf9d7e047ea818c7d12c79dcac1d3e5aa979329b --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4296e644b36d9e3925a9a864e3e6dcd3aad334baf0d748dbf4c697fa10575ff2 +size 1096196 diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45e0d6bfbee4547db4c920dfc6d8502c1cf671ad --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09f9d609000e258b620335c623ae25f739a95ae910ce21f6a01bbb2442d21fa5 +size 680759 diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e2545aec4571fe8b7ef8324afee81f76a814223 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b814eef54727e0c0f4acb8be2cd0ad22d83cb7c3153b4deb14d42fe1cee7428 +size 1364301 diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aacb89ee5c4aff8031b39e85568daa941104c764 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff0c295868f146e8406664ee1dbe5f8d174a53efdbc91e2590c0ecbebe7e64c6 +size 1329769 diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..37709c271b725a0dbffe94f4911abad9889115a7 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log @@ -0,0 +1,291 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736298685.533765]: resetting environment +[INFO] [1736298685.538574]: cleared current plan +[INFO] [1736298685.539485]: moving to home +[INFO] [1736298686.541467]: reached home +[INFO] [1736298686.542717]: reset action history +[INFO] [1736298689.258672]: environment reset +Starting episode 0 +[INFO] [1736298689.263706]: resetting environment +[INFO] [1736298689.267418]: cleared current plan +[INFO] [1736298689.268557]: moving to home +[INFO] [1736298690.270926]: reached home +[INFO] [1736298690.272335]: reset action history +[INFO] [1736298692.982333]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736298697.772874]: resetting environment +[INFO] [1736298697.779824]: cleared current plan +[INFO] [1736298697.780766]: moving to home +[INFO] [1736298698.782864]: reached home +[INFO] [1736298698.784749]: reset action history +[INFO] [1736298701.494012]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/episode_rosbags/episode_0_2025-01-07-20-11-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2221061.log +started roslaunch server http://158.130.50.37:32959/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2221590] +process[print_text-2]: started with pid [2221614] +process[rosbag_record-3]: started with pid [2221615] +started bagging! +For topic gripper_width: timestamp difference is 101128837 for nearest: 1736298718671551248 - target: 1736298718570422411 at idx 224 +preempting episode +[INFO] [1736298727.933204]: Storing episode... +[print_text-2] killing on exit +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1736298728.721222]: Stored episode 1. +[INFO] [1736298728.721429]: Saving video... +[INFO] [1736298728.940154]: Video saved! +Starting episode 1 +[INFO] [1736298728.956785]: resetting environment +[INFO] [1736298728.966356]: cleared current plan +[INFO] [1736298728.967165]: moving to home +[INFO] [1736298732.469202]: reached home +[INFO] [1736298732.469632]: reset action history +[INFO] [1736298735.185282]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736298749.455647]: resetting environment +[INFO] [1736298749.462066]: cleared current plan +[INFO] [1736298749.462976]: moving to home +[INFO] [1736298750.464643]: reached home +[INFO] [1736298750.465057]: reset action history +[INFO] [1736298753.224159]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/episode_rosbags/episode_1_2025-01-07-20-12-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2221061.log +started roslaunch server http://158.130.50.37:40221/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2221743] +process[print_text-5]: started with pid [2221744] +process[rosbag_record-6]: started with pid [2221745] +started bagging! +For topic gripper_width: timestamp difference is 100205131 for nearest: 1736298761469341703 - target: 1736298761369136572 at idx 118 +For topic gripper_width: timestamp difference is 109413896 for nearest: 1736298772969346795 - target: 1736298772859932899 at idx 249 +preempting episode +[INFO] [1736298779.118796]: Storing episode... +[print_text-5] killing on exit[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736298779.839595]: Stored episode 2. +[INFO] [1736298779.840521]: Saving video... +[INFO] [1736298780.439775]: Video saved! +Starting episode 2 +[INFO] [1736298780.462562]: resetting environment +[INFO] [1736298780.480323]: cleared current plan +[INFO] [1736298780.481709]: moving to home +[WARN] [1736298780.284915]: Plan exhausted +[WARN] [1736298780.323349]: Plan exhausted +[WARN] [1736298780.363346]: Plan exhausted +[WARN] [1736298780.405864]: Plan exhausted +[WARN] [1736298780.444083]: Plan exhausted +[INFO] [1736298784.187437]: reached home +[INFO] [1736298784.189487]: reset action history +[INFO] [1736298786.960171]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736298793.725845]: resetting environment +[INFO] [1736298793.733187]: cleared current plan +[INFO] [1736298793.733981]: moving to home +[INFO] [1736298794.735822]: reached home +[INFO] [1736298794.736657]: reset action history +[INFO] [1736298797.445237]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/episode_rosbags/episode_2_2025-01-07-20-13-17.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2221061.log +started roslaunch server http://158.130.50.37:35413/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2221888] +process[print_text-8]: started with pid [2221889] +process[rosbag_record-9]: started with pid [2221913] +started bagging! +preempting episode +[INFO] [1736298811.831580]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736298812.650340]: Stored episode 3. +[INFO] [1736298812.651853]: Saving video... +[WARN] [1736298812.965236]: Plan exhausted +[WARN] [1736298813.003994]: Plan exhausted +[WARN] [1736298813.047087]: Plan exhausted +[WARN] [1736298813.088911]: Plan exhausted +[WARN] [1736298813.123479]: Plan exhausted +[WARN] [1736298813.165792]: Plan exhausted +[WARN] [1736298813.206540]: Plan exhausted +[WARN] [1736298813.247069]: Plan exhausted +[WARN] [1736298813.283540]: Plan exhausted +[INFO] [1736298813.243336]: Video saved! +Starting episode 3 +[INFO] [1736298813.281995]: resetting environment +[INFO] [1736298813.301677]: cleared current plan +[INFO] [1736298813.304480]: moving to home +[INFO] [1736298816.813217]: reached home +[INFO] [1736298816.816225]: reset action history +[INFO] [1736298819.717097]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736298830.249003]: resetting environment +[INFO] [1736298830.257577]: cleared current plan +[INFO] [1736298830.257763]: moving to home +[INFO] [1736298831.259784]: reached home +[INFO] [1736298831.261633]: reset action history +[INFO] [1736298833.989053]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/episode_rosbags/episode_3_2025-01-07-20-13-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2221061.log +started roslaunch server http://158.130.50.37:43679/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2222031] +process[print_text-11]: started with pid [2222032] +process[rosbag_record-12]: started with pid [2222056] +started bagging! +For topic gripper_width: timestamp difference is -110187068 for nearest: 1736298839502677903 - target: 1736298839612864971 at idx 68 +[INFO] [1736298868.769341]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1736298869.568223]: Stored episode 4. +[INFO] [1736298869.576086]: Saving video... +[INFO] [1736298870.018782]: Video saved! +Starting episode 4 +[INFO] [1736298870.097026]: resetting environment +[INFO] [1736298870.122557]: cleared current plan +[INFO] [1736298870.128098]: moving to home +[INFO] [1736298873.641809]: reached home +[INFO] [1736298873.649127]: reset action history +[INFO] [1736298876.395238]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736298883.102686]: resetting environment +[INFO] [1736298883.112616]: cleared current plan +[INFO] [1736298883.112816]: moving to home +[INFO] [1736298884.115824]: reached home +[INFO] [1736298884.119619]: reset action history +[INFO] [1736298886.891510]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/episode_rosbags/episode_4_2025-01-07-20-14-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2221061.log +started roslaunch server http://158.130.50.37:39007/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2222174] +process[print_text-14]: started with pid [2222175] +process[rosbag_record-15]: started with pid [2222199] +started bagging! +For topic gripper_width: timestamp difference is 107654624 for nearest: 1736298894669354968 - target: 1736298894561700344 at idx 111 +preempting episode +[INFO] [1736298918.845862]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736298919.612143]: Stored episode 5. +[INFO] [1736298919.615366]: Saving video... +[INFO] [1736298920.178512]: Video saved! +[INFO] [1736298920.229889]: resetting environment +[INFO] [1736298920.269153]: cleared current plan +[INFO] [1736298920.271830]: moving to home +[WARN] [1736298919.964330]: Plan exhausted +[WARN] [1736298920.005935]: Plan exhausted +[WARN] [1736298920.043417]: Plan exhausted +[WARN] [1736298920.084651]: Plan exhausted +[WARN] [1736298920.124185]: Plan exhausted +[WARN] [1736298920.164268]: Plan exhausted +[WARN] [1736298920.203606]: Plan exhausted +[WARN] [1736298920.243437]: Plan exhausted +[INFO] [1736298923.977268]: reached home +[INFO] [1736298923.988145]: reset action history +[INFO] [1736298926.791181]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/requirements.txt b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-metadata.json b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ff53799a5c2e0b7fed2fe94c418f8266e700e4a7 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:11:20.973511", + "startedAt": "2025-01-08T01:11:20.653238", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3560.282, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4466.498, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.02, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.992, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 554.1286163330078 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..94cc15cb522647cff2fdd693960532ac35b5cdb3 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c4ea3df26327a48d1e752e90490983d710c64c6eb48ced1b5717e657466563f0", "size": 1163698, "path": "media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4"}, "global_step": 506, "_timestamp": 1736298932.1198459, "_runtime": 251.45767092704773, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "4296e644b36d9e3925a9a864e3e6dcd3aad334baf0d748dbf4c697fa10575ff2", "size": 1096196, "path": "media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4"}, "eval/num_success": 0.0, "episode": 60.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "09f9d609000e258b620335c623ae25f739a95ae910ce21f6a01bbb2442d21fa5", "size": 680759, "path": "media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "6b814eef54727e0c0f4acb8be2cd0ad22d83cb7c3153b4deb14d42fe1cee7428", "size": 1364301, "path": "media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ff0c295868f146e8406664ee1dbe5f8d174a53efdbc91e2590c0ecbebe7e64c6", "size": 1329769, "path": "media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://q8khfpuf8k4fiadlt5cppfvssarzjbw5zb3pf5p6r6ukhd7mjm90d2xntcc6uz5d9ezbw96zg9d9f6pt1j7gt3lc7y8ilz58u93mj2bevsfwds1d2k326ba2ls57cgcp/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://moi3gtpwgemeeymzrlppkwaczwr1mj60drlu2vutzt7egi1tvh8lxp8594lvjlcsaz5qzeqg6ujkel4mbghkp9zlz2fdvec1nz1einxd8dqa4274fjofw3fxvxd7ycqo:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 37, "total_num_episodes": 60, "total_success_rate": 0.6166666666666667, "total_success_rate_lb_0.01": 0.46349006891980815, "total_success_rate_ub_0.01": 0.7536167203872801, "total_success_rate_lb_0.025": 0.4929127618684639, "total_success_rate_ub_0.025": 0.7386003359675044, "total_success_rate_lb_0.05": 0.5128696039293299, "total_success_rate_ub_0.05": 0.7125004790298323, "total_success_rate_lb_0.1": 0.5289849340857711, "total_success_rate_ub_0.1": 0.6996142845201615, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "df810a6369b5406d344dd2f3be306fd45d9aa7e8ff4943ef69757e5056e3dc32", "size": 521, "artifact_path": "wandb-client-artifact://0rdfll66dslfbpvy83739sgpxmzbjresfp96ljsjnol9tyy9190l39tkqdmemlkzfcwhtdor2k2azoffbwc9id6dbpn0rprzy2zjr0tvbafcgih6ikzjc2auoxebnp88/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zpzz0eel11kvbn4zhl985jx5e4pkbgjk2iiqmxfmezkfno4fzxm98olc2d4tgu9gk52ehstbufis0yvob1hhde3vqun13ig3ljhfhyyjnslgudbef8j5tm3lz2rtgbjf:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 262}, "eval/total_success_rate": 0.6166666746139526, "eval/total_success_rate_lb_0.01": 0.4634900689125061, "eval/total_success_rate_ub_0.01": 0.7536166906356812, "eval/total_success_rate_lb_0.025": 0.49291276931762695, "eval/total_success_rate_ub_0.025": 0.7386003136634827, "eval/total_success_rate_lb_0.05": 0.5128695964813232, "eval/total_success_rate_ub_0.05": 0.7125004529953003, "eval/total_success_rate_lb_0.1": 0.5289849042892456, "eval/total_success_rate_ub_0.1": 0.6996142864227295} \ No newline at end of file diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug-internal.log b/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bd5496c5da5b7e43b8d494d79da2322044f03e4b --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug-internal.log @@ -0,0 +1,756 @@ +2025-01-07 20:11:20,661 INFO StreamThr :2221501 [internal.py:wandb_internal():85] W&B internal server running at pid: 2221501, started at: 2025-01-07 20:11:20.660615 +2025-01-07 20:11:20,661 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:11:20,675 INFO WriterThread:2221501 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/run-urh96fgp.wandb +2025-01-07 20:11:20,675 DEBUG SenderThread:2221501 [sender.py:send():379] send: header +2025-01-07 20:11:20,688 DEBUG SenderThread:2221501 [sender.py:send():379] send: run +2025-01-07 20:11:20,894 INFO SenderThread:2221501 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files +2025-01-07 20:11:20,894 INFO SenderThread:2221501 [sender.py:_start_run_threads():1188] run started: urh96fgp with start time 1736298680.662175 +2025-01-07 20:11:20,911 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:11:20,912 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:11:20,955 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:11:20,960 DEBUG HandlerThread:2221501 [system_info.py:__init__():26] System info init +2025-01-07 20:11:20,960 DEBUG HandlerThread:2221501 [system_info.py:__init__():41] System info init done +2025-01-07 20:11:20,960 INFO HandlerThread:2221501 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:11:20,960 INFO SystemMonitor:2221501 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:11:20,960 INFO HandlerThread:2221501 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:11:20,961 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:11:20,961 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:11:20,962 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:11:20,962 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:11:20,963 INFO SystemMonitor:2221501 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:11:20,973 DEBUG HandlerThread:2221501 [system_info.py:probe():152] Probing system +2025-01-07 20:11:20,974 DEBUG HandlerThread:2221501 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:11:20,979 DEBUG HandlerThread:2221501 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:11:20,979 DEBUG HandlerThread:2221501 [system_info.py:probe():200] Probing system done +2025-01-07 20:11:20,979 DEBUG HandlerThread:2221501 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:11:20.973511', 'startedAt': '2025-01-08T01:11:20.653238', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.282, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4466.498, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.02, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.992, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 554.1286163330078}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:11:20,979 INFO HandlerThread:2221501 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:11:20,979 INFO HandlerThread:2221501 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:11:20,979 DEBUG HandlerThread:2221501 [system_info.py:_save_code():44] Saving code +2025-01-07 20:11:20,981 DEBUG HandlerThread:2221501 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:11:20,981 DEBUG HandlerThread:2221501 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:11:21,010 DEBUG HandlerThread:2221501 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:11:21,010 INFO HandlerThread:2221501 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:11:21,014 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:11:21,014 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:11:21,014 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:11:21,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:11:21,267 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:11:21,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:21,269 DEBUG SenderThread:2221501 [sender.py:send():379] send: telemetry +2025-01-07 20:11:21,289 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/rxji41gk-wandb-metadata.json +2025-01-07 20:11:21,293 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/f4kmqkjd-code/FISH/eval_robot.py +2025-01-07 20:11:21,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:11:21,309 INFO HandlerThread:2221501 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/tb" + save: true +} +control { + relay_id: "b30f97c685eb4f2885a4012e0da1b4ce" +} +_info { + stream_id: "urh96fgp" +} + +2025-01-07 20:11:21,309 DEBUG HandlerThread:2221501 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:11:21,310 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:11:21,448 DEBUG SenderThread:2221501 [sender.py:send():379] send: tbrecord +2025-01-07 20:11:21,449 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:11:21,449 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 with policy live +2025-01-07 20:11:21,641 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/d8m4ti26-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:11:21,896 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/requirements.txt +2025-01-07 20:11:21,896 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:11:21,896 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:21,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code/FISH/eval_robot.py +2025-01-07 20:11:21,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-metadata.json +2025-01-07 20:11:21,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code/FISH +2025-01-07 20:11:21,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code +2025-01-07 20:11:22,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:23,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:23,895 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:24,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:25,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:25,533 DEBUG SenderThread:2221501 [sender.py:send():379] send: config +2025-01-07 20:11:26,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:26,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:27,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:27,897 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:28,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:29,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:30,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:31,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:31,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:31,898 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:32,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:33,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:34,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:35,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:35,899 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:36,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:11:36,267 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:11:36,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:36,330 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:37,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:38,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:39,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:39,900 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:40,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:41,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:41,494 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:42,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:43,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:43,901 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:11:44,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:45,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:46,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:47,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:47,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:48,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:49,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:50,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:51,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:11:51,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:11:51,313 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:52,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:52,406 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:52,903 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml +2025-01-07 20:11:53,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:54,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:55,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:56,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:57,274 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:58,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:11:58,274 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:59,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:11:59,905 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:00,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:01,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:02,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:03,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:04,269 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:04,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:05,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:06,267 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:12:06,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:12:06,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:07,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:08,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:08,955 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:12:08,955 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:12:08,955 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 with policy now +2025-01-07 20:12:08,956 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:12:08,956 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:12:08,958 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:12:09,206 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/tqh7i2v8-media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 +2025-01-07 20:12:09,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media +2025-01-07 20:12:09,908 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos +2025-01-07 20:12:09,967 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:10,213 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/gyo0vgzq-media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 +2025-01-07 20:12:10,278 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:11,278 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:12,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:13,278 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:13,910 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:14,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:15,185 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:15,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:16,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:17,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:17,911 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:18,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:19,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:20,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:20,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:20,963 DEBUG SystemMonitor:2221501 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:12:20,963 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:12:21,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:12:21,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:12:21,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:22,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:23,279 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:24,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:25,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:25,352 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:26,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:27,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:28,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:29,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:29,915 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:12:30,177 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/06kf33l8-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:12:30,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:30,465 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:31,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:31,916 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:32,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:33,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:34,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:35,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:35,466 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:35,917 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:36,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:12:36,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:12:36,313 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:37,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:38,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:39,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:40,281 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:41,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:41,282 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:42,282 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:43,282 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:43,920 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:44,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:45,283 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:46,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:46,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:47,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:48,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:49,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:50,285 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:50,964 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:12:51,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:12:51,268 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:12:51,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:51,415 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:52,285 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:53,285 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:53,923 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:12:54,285 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:55,293 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:56,293 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:57,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:12:57,293 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:58,293 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:12:59,294 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:00,297 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:00,457 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:13:00,457 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 with policy now +2025-01-07 20:13:00,460 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:13:00,461 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:13:00,461 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:13:00,461 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:13:00,699 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/hz4ko86e-media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 +2025-01-07 20:13:00,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:13:00,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 +2025-01-07 20:13:00,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:13:01,198 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/m3177g2j-media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 +2025-01-07 20:13:01,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:01,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:02,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:02,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:02,925 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:03,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:04,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:05,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:05,926 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:06,277 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:13:06,277 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:13:06,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:07,298 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:07,927 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:08,005 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:08,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:09,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:10,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:11,299 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:12,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:13,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:13,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:13,929 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:13:14,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:14,716 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:13:14,716 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:13:14,716 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:13:14,929 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:13:15,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:15,930 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:16,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:17,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:18,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:18,697 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:19,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:19,931 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:20,300 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:20,965 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:13:21,268 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:13:21,269 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:13:21,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:22,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:23,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:24,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:24,357 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:25,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:26,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:27,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:28,301 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:29,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:30,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:30,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:31,302 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:32,305 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:33,278 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:13:33,278 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 with policy now +2025-01-07 20:13:33,280 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:13:33,281 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:13:33,281 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:13:33,282 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:13:33,307 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:33,553 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/ur6qagdb-media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 +2025-01-07 20:13:33,935 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:13:33,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:33,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 +2025-01-07 20:13:33,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:13:34,197 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/xh7y27ox-media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 +2025-01-07 20:13:34,310 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:34,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:35,305 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:35,309 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:35,936 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:36,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:13:36,276 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:13:36,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:37,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:37,937 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:38,319 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:39,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:40,315 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:40,761 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:41,315 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:41,938 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:42,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:43,314 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:44,314 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:45,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:46,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:46,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:47,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:48,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:49,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:50,315 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:50,539 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:13:50,539 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:13:50,550 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:13:50,941 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:13:50,942 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:13:50,965 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:13:51,224 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/uev3j49l-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:13:51,305 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:13:51,305 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:51,306 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:13:51,325 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:51,942 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:52,322 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:53,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:53,942 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:54,315 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:55,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:55,943 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:13:56,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:57,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:13:57,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:58,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:13:59,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:00,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:01,316 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:01,945 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:02,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:03,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:03,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:04,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:05,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:06,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:14:06,270 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:14:06,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:07,317 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:08,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:08,330 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:09,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:10,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:11,318 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:12,319 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:13,319 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:14,273 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:14,319 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:15,320 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:16,320 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:17,320 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:18,320 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:19,275 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:19,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:20,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:20,965 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:14:21,270 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:14:21,270 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:14:21,321 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:22,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:23,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:24,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:24,364 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:25,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:26,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:27,323 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:28,324 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:29,333 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:29,568 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:29,955 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:30,076 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:14:30,076 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 with policy now +2025-01-07 20:14:30,082 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:14:30,083 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:14:30,083 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:14:30,088 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:14:30,328 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:30,347 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/m8z85ohs-media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 +2025-01-07 20:14:30,956 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:14:30,956 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 +2025-01-07 20:14:30,956 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:14:31,245 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/zl5hyo3l-media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 +2025-01-07 20:14:31,332 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:31,956 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:32,326 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:33,331 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:34,331 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:34,689 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:35,332 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:35,957 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:36,284 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:14:36,284 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:14:36,333 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:37,334 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:37,958 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:38,340 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:39,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:40,336 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:40,434 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:41,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:42,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:43,337 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:43,603 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:14:43,603 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:14:43,604 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:14:43,960 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:14:43,960 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:43,960 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:14:44,339 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:45,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:45,961 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:46,157 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:46,347 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:47,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:47,961 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:48,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:49,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:50,341 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:50,966 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:14:51,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:14:51,271 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:14:51,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:51,385 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:52,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:53,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:54,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:55,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:55,964 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:14:56,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:57,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:14:57,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:58,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:14:59,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:00,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:01,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:02,276 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:02,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:03,344 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:04,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:05,344 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:06,271 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:15:06,271 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:15:06,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:07,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:07,420 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:08,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:09,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:10,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:11,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:12,345 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:13,277 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:13,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:14,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:15,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:16,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:17,346 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:18,347 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:18,842 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:19,351 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:19,972 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:20,224 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:15:20,224 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 with policy now +2025-01-07 20:15:20,228 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:15:20,228 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:20,229 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:20,229 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:20,349 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:20,500 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/dubhrrmg-media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 +2025-01-07 20:15:20,972 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:20,973 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 +2025-01-07 20:15:20,973 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval +2025-01-07 20:15:21,006 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:15:21,291 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:15:21,291 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:15:21,297 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/mnpvnpde-media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 +2025-01-07 20:15:21,355 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:21,973 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:22,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:22,973 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:23,357 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:23,977 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:24,356 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:25,358 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:25,974 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:26,364 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:27,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:27,975 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:28,375 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:29,362 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:29,837 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:30,365 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:31,364 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:31,628 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:15:31,628 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:15:31,803 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:15:31,976 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:31,976 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:31,976 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:15:31,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media +2025-01-07 20:15:31,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table +2025-01-07 20:15:31,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval +2025-01-07 20:15:32,155 INFO wandb-upload_1:2221501 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpezyg0hgw +2025-01-07 20:15:32,188 INFO wandb-upload_2:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/bkk4uymf-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:15:32,194 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:15:32,204 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:15:32,261 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/sp33j6e1-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:32,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:32,551 INFO SenderThread:2221501 [sender.py:send_request_log_artifact():1525] logged artifact run-urh96fgp-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM2NjY4NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyNjY5Nw==', 'latestArtifact': None}} +2025-01-07 20:15:32,551 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:32,551 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,551 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,552 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:15:32,552 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:15:32,552 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:32,552 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,555 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,556 DEBUG SenderThread:2221501 [sender.py:send():379] send: telemetry +2025-01-07 20:15:32,556 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,556 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,556 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,557 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,557 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,557 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,557 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,557 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,558 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,558 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,558 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,558 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,558 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,559 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,559 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,559 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,559 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,560 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,560 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,560 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,560 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:32,561 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:32,561 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:15:32,806 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/ybong3xl-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:15:32,952 INFO wandb-upload_0:2221501 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1yq2keh3 +2025-01-07 20:15:32,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:32,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:32,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json +2025-01-07 20:15:32,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval +2025-01-07 20:15:33,318 INFO SenderThread:2221501 [sender.py:send_request_log_artifact():1525] logged artifact run-urh96fgp-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM2NjcyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyNjcwMg==', 'latestArtifact': None}} +2025-01-07 20:15:33,320 DEBUG SenderThread:2221501 [sender.py:send():379] send: files +2025-01-07 20:15:33,320 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json with policy now +2025-01-07 20:15:33,321 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:33,321 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:33,322 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:33,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:33,523 INFO wandb-upload_2:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/yg19ecs8-media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json +2025-01-07 20:15:33,772 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/wcxjbygd-media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json +2025-01-07 20:15:33,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:33,977 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:34,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:35,322 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:35,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:36,272 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:15:36,272 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:15:36,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:37,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:38,360 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:39,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:40,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:40,410 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:40,980 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml +2025-01-07 20:15:41,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:42,361 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:43,285 DEBUG SenderThread:2221501 [sender.py:send():379] send: exit +2025-01-07 20:15:43,285 INFO SenderThread:2221501 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:15:43,285 INFO SenderThread:2221501 [sender.py:send_exit():588] handling runtime: 262 +2025-01-07 20:15:43,286 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:send_exit():594] send defer +2025-01-07 20:15:43,287 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,287 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:15:43,287 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:15:43,287 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,287 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:15:43,287 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:15:43,287 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:15:43,287 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,287 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:15:43,288 INFO HandlerThread:2221501 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:15:43,288 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:15:43,288 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:15:43,288 DEBUG SystemMonitor:2221501 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:15:43,288 DEBUG SystemMonitor:2221501 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:15:43,341 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:15:43,342 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:15:43,342 INFO HandlerThread:2221501 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:15:43,342 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:43,342 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:15:43,342 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:15:43,342 DEBUG SenderThread:2221501 [sender.py:send():379] send: stats +2025-01-07 20:15:43,342 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,343 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:15:43,343 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:43,343 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:15:43,343 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:15:43,343 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:43,343 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:15:43,981 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:43,981 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:49,663 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:49,664 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:49,664 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:15:49,665 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:49,665 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:15:49,665 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:15:49,665 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:49,665 DEBUG SenderThread:2221501 [sender.py:send():379] send: history +2025-01-07 20:15:49,666 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:15:49,666 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:49,666 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:15:49,667 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:49,667 DEBUG SenderThread:2221501 [sender.py:send():379] send: summary +2025-01-07 20:15:49,667 INFO SenderThread:2221501 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:15:49,667 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:49,667 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:15:49,668 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:15:49,668 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:49,668 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:15:49,668 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:49,668 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:15:49,668 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:15:49,668 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:15:49,668 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:49,668 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:15:49,668 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:49,669 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:15:49,984 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:50,286 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:50,457 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:15:50,457 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:50,457 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:50,457 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:15:50,458 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:50,458 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:15:50,458 INFO SenderThread:2221501 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:15:50,460 INFO SenderThread:2221501 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:15:50,462 INFO SenderThread:2221501 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:15:50,475 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:15:50,475 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:50,475 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:15:50,476 DEBUG SenderThread:2221501 [sender.py:send():379] send: artifact +2025-01-07 20:15:50,781 INFO SenderThread:2221501 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:15:50,782 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:50,782 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:15:50,782 INFO SenderThread:2221501 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:15:50,984 INFO Thread-12 :2221501 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:50,985 INFO SenderThread:2221501 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files +2025-01-07 20:15:50,985 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml config.yaml +2025-01-07 20:15:50,985 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json wandb-summary.json +2025-01-07 20:15:50,985 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:50,988 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/requirements.txt requirements.txt +2025-01-07 20:15:50,988 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:15:50,988 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log output.log +2025-01-07 20:15:50,988 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 media/videos/eval/4_eval_4_ff0c295868f146e84066.mp4 +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 media/videos/eval/1_eval_1_4296e644b36d9e3925a9.mp4 +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 media/videos/eval/3_eval_3_6b814eef54727e0c0f4a.mp4 +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 media/videos/eval/2_eval_2_09f9d609000e258b6203.mp4 +2025-01-07 20:15:50,990 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 media/videos/eval/0_eval_0_c4ea3df26327a48d1e75.mp4 +2025-01-07 20:15:50,993 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json media/table/eval/total_success_rate_ci_6_df810a6369b5406d344d.table.json +2025-01-07 20:15:50,993 INFO SenderThread:2221501 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:15:50,993 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:15:50,993 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:50,993 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:15:50,994 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:50,994 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:15:50,994 INFO SenderThread:2221501 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:15:51,160 INFO wandb-upload_2:2221501 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/config.yaml +2025-01-07 20:15:51,164 INFO wandb-upload_1:2221501 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/wandb-summary.json +2025-01-07 20:15:51,210 INFO wandb-upload_3:2221501 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/requirements.txt +2025-01-07 20:15:51,239 INFO wandb-upload_4:2221501 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/files/output.log +2025-01-07 20:15:51,243 INFO wandb-upload_0:2221501 [upload_job.py:push():130] Uploaded file /tmp/tmpq3wiyjefwandb/kcrplpt4-events.out.tfevents.1736298681.leonmkim-ROG-Strix-G15CS-G15CS.2221061.0 +2025-01-07 20:15:51,287 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:51,287 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:51,443 INFO Thread-11 (_thread_body):2221501 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:15:51,443 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:51,443 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:15:51,443 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:51,443 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:15:51,443 INFO SenderThread:2221501 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:15:51,445 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:15:51,445 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:51,445 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:15:51,445 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:51,445 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:15:51,445 INFO SenderThread:2221501 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:15:51,674 INFO SenderThread:2221501 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:15:51,674 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:15:51,674 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:51,674 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:15:51,674 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:51,674 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:15:51,674 INFO SenderThread:2221501 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:15:51,674 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:15:51,674 INFO HandlerThread:2221501 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:15:51,675 DEBUG SenderThread:2221501 [sender.py:send():379] send: final +2025-01-07 20:15:51,675 DEBUG SenderThread:2221501 [sender.py:send():379] send: footer +2025-01-07 20:15:51,675 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: defer +2025-01-07 20:15:51,675 INFO SenderThread:2221501 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:15:51,675 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:51,675 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:15:51,676 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:15:51,676 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:15:51,676 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:51,676 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:15:51,677 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:15:51,677 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:15:51,737 DEBUG SenderThread:2221501 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:15:51,737 INFO MainThread:2221501 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:15:51,738 INFO MainThread:2221501 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:15:51,738 INFO MainThread:2221501 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:15:51,738 DEBUG HandlerThread:2221501 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:15:51,738 INFO HandlerThread:2221501 [handler.py:finish():882] shutting down handler +2025-01-07 20:15:52,676 INFO WriterThread:2221501 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/run-urh96fgp.wandb +2025-01-07 20:15:52,737 INFO SenderThread:2221501 [sender.py:finish():1615] shutting down sender +2025-01-07 20:15:52,737 INFO SenderThread:2221501 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:15:52,737 INFO SenderThread:2221501 [file_pusher.py:join():175] waiting for file pusher diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug.log b/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d96ce063879470bc557fe430ae878acb2c761282 --- /dev/null +++ b/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Configure stats pid to 2221061 +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/settings +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug.log +2025-01-07 20:11:20,655 INFO MainThread:2221061 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/wandb/run-20250107_201120-urh96fgp/logs/debug-internal.log +2025-01-07 20:11:20,656 INFO MainThread:2221061 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:11:20,656 INFO MainThread:2221061 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp'} +2025-01-07 20:11:20,656 INFO MainThread:2221061 [wandb_init.py:init():619] starting backend +2025-01-07 20:11:20,656 INFO MainThread:2221061 [wandb_init.py:init():623] setting up manager +2025-01-07 20:11:20,659 INFO MainThread:2221061 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:11:20,661 INFO MainThread:2221061 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:11:20,675 INFO MainThread:2221061 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:11:20,682 INFO MainThread:2221061 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:11:20,911 INFO MainThread:2221061 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:11:20,949 INFO MainThread:2221061 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:11:20,950 INFO MainThread:2221061 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:11:21,267 INFO MainThread:2221061 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:11:21,268 INFO MainThread:2221061 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:11:21,268 INFO MainThread:2221061 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:11:21,268 INFO MainThread:2221061 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:11:21,270 INFO MainThread:2221061 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:11:21,270 INFO MainThread:2221061 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/urh96fgp/tb, True +2025-01-07 20:11:25,532 INFO MainThread:2221061 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:15:52,739 WARNING MsgRouterThr:2221061 [router.py:message_loop():77] message_loop has been closed diff --git a/urh96fgp/wandb/run-20250107_201120-urh96fgp/run-urh96fgp.wandb b/urh96fgp/wandb/run-20250107_201120-urh96fgp/run-urh96fgp.wandb new file mode 100644 index 0000000000000000000000000000000000000000..64c9a39814cfe434eb60a81bf832962f05481ff0 Binary files /dev/null and b/urh96fgp/wandb/run-20250107_201120-urh96fgp/run-urh96fgp.wandb differ