diff --git a/.gitattributes b/.gitattributes index 57800db0c512024f59a8f294a9e3383661d39cb7..d8314634079cd07e1eca3910aef0fa3dae461df9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -119,3 +119,36 @@ nyyc47dd/episode_rosbags/episode_0_2025-01-07-17-10-04.bag filter=lfs diff=lfs m 9fviy2t8/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text 9fviy2t8/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text 9jw2ghih/episode_rosbags/episode_0_2025-01-06-20-41-14.bag filter=lfs diff=lfs merge=lfs -text +9fviy2t8/episode_rosbags/episode_4_2025-01-06-17-27-09.bag filter=lfs diff=lfs merge=lfs -text +9fviy2t8/episode_rosbags/episode_3_2025-01-06-17-26-27.bag filter=lfs diff=lfs merge=lfs -text +nyyc47dd/episode_rosbags/episode_1_2025-01-07-17-10-47.bag filter=lfs diff=lfs merge=lfs -text +5qmtbx9d/episode_rosbags/episode_1_2025-01-07-20-46-02.bag filter=lfs diff=lfs merge=lfs -text +o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 filter=lfs diff=lfs merge=lfs -text +w3dz1sd1/episode_rosbags/episode_3_2025-01-07-17-48-43.bag filter=lfs diff=lfs merge=lfs -text +o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 filter=lfs diff=lfs merge=lfs -text +o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 filter=lfs diff=lfs merge=lfs -text +o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 filter=lfs diff=lfs merge=lfs -text +o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 filter=lfs diff=lfs merge=lfs -text +w3dz1sd1/episode_rosbags/episode_4_2025-01-07-17-49-25.bag filter=lfs diff=lfs merge=lfs -text +o9qcewu3/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +o9qcewu3/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +o9qcewu3/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +o9qcewu3/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +o9qcewu3/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +9jw2ghih/episode_rosbags/episode_4_2025-01-06-20-43-59.bag filter=lfs diff=lfs merge=lfs -text +o9qcewu3/episode_rosbags/episode_1_2025-01-07-20-50-59.bag filter=lfs diff=lfs merge=lfs -text +o9qcewu3/episode_rosbags/episode_0_2025-01-07-20-50-16.bag filter=lfs diff=lfs merge=lfs -text +o9qcewu3/episode_rosbags/episode_2_2025-01-07-20-51-45.bag filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +o9qcewu3/episode_rosbags/episode_3_2025-01-07-20-52-32.bag filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/episode_rosbags/episode_4_2025-01-06-22-12-15.bag filter=lfs diff=lfs merge=lfs -text +5jdyg1q2/episode_rosbags/episode_2_2025-01-06-22-10-54.bag filter=lfs diff=lfs merge=lfs -text diff --git a/5jdyg1q2/.hydra/config.yaml b/5jdyg1q2/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/5jdyg1q2/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/5jdyg1q2/.hydra/hydra.yaml b/5jdyg1q2/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7765ceb969738ef45b156e2b2ffac1007258d5ef --- /dev/null +++ b/5jdyg1q2/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/5jdyg1q2/.hydra/overrides.yaml b/5jdyg1q2/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/5jdyg1q2/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/5jdyg1q2/episode_rosbags/aligned_depth_to_color_K.npy b/5jdyg1q2/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5jdyg1q2/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5jdyg1q2/episode_rosbags/cam_tf_world.npy b/5jdyg1q2/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/5jdyg1q2/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/5jdyg1q2/episode_rosbags/color_K.npy b/5jdyg1q2/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5jdyg1q2/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5jdyg1q2/episode_rosbags/depth_K.npy b/5jdyg1q2/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/5jdyg1q2/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/5jdyg1q2/episode_rosbags/episode_2_2025-01-06-22-10-54.bag b/5jdyg1q2/episode_rosbags/episode_2_2025-01-06-22-10-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..cc044ccba9cb2c6042d286e32c87796e00ef4439 --- /dev/null +++ b/5jdyg1q2/episode_rosbags/episode_2_2025-01-06-22-10-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be985db442925064e0e347bca3bde1b61eff4fa1dd98ed8d89735ecfcee01a15 +size 1413083873 diff --git a/5jdyg1q2/episode_rosbags/episode_4_2025-01-06-22-12-15.bag b/5jdyg1q2/episode_rosbags/episode_4_2025-01-06-22-12-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..506cceaacef60c491a83943b6507f54a749a8f96 --- /dev/null +++ b/5jdyg1q2/episode_rosbags/episode_4_2025-01-06-22-12-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc86d789998764c931318ba0298c34ccfb21aa351bcfaa273844d6ea7880cccb +size 1408844691 diff --git a/5jdyg1q2/eval_robot.log b/5jdyg1q2/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..7346c3afe9a8fffe00edeaab953f6df33b5bba4b --- /dev/null +++ b/5jdyg1q2/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 22:08:59,771][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:08:59,774][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/5jdyg1q2/eval_video/0_eval.mp4 b/5jdyg1q2/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7810ac4a4e95bcf1f6165bed0226d89a697d730e --- /dev/null +++ b/5jdyg1q2/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6426b8e8aadca81f0c9e61bd8f64f1578e6f4f97caef958186eed7e5f2d31306 +size 1145928 diff --git a/5jdyg1q2/eval_video/1_eval.mp4 b/5jdyg1q2/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b1d941e06fd48491d4f5b85e3fec1eeadf40e7ff --- /dev/null +++ b/5jdyg1q2/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6e43c5f90cc9d422ca7e5b4677d6816f44b66e7a4a4594bffe010680ec0994 +size 1146021 diff --git a/5jdyg1q2/eval_video/2_eval.mp4 b/5jdyg1q2/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df52b46c8bd1283cb1de1c8792d267619d8ecd32 --- /dev/null +++ b/5jdyg1q2/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edc3f417d98fcdf1019e50974244ca5e7dc844b3fb6a9fcbfe608cb43e538275 +size 1139041 diff --git a/5jdyg1q2/eval_video/3_eval.mp4 b/5jdyg1q2/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba757932e4150f86d52a9606db762954ae10c91e --- /dev/null +++ b/5jdyg1q2/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc101df27963f59b97f2dec5c530041fcb86770368f574b840b6d71511b01799 +size 1141420 diff --git a/5jdyg1q2/eval_video/4_eval.mp4 b/5jdyg1q2/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..791d1c5fff05fe90f5d22e5b7ec81991cf2b1b13 --- /dev/null +++ b/5jdyg1q2/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9c235f657b8c4c13ec23d5ad66221e2a5888753182f820a13a8677a3f3ed0cf +size 1138718 diff --git a/5jdyg1q2/tb/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 b/5jdyg1q2/tb/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 new file mode 100644 index 0000000000000000000000000000000000000000..9fdf504b85a4de188f222c789b845af52cd8609e --- /dev/null +++ b/5jdyg1q2/tb/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18a04eb79970f70fe9c79458df057d872cdb4c29c92f80b45e823656acb1e727 +size 6539 diff --git a/5jdyg1q2/wandb/debug-internal.log b/5jdyg1q2/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1250fecfa6c3bafe0633cfd790dc2d07192e710f --- /dev/null +++ b/5jdyg1q2/wandb/debug-internal.log @@ -0,0 +1,716 @@ +2025-01-06 22:09:05,581 INFO StreamThr :2121135 [internal.py:wandb_internal():85] W&B internal server running at pid: 2121135, started at: 2025-01-06 22:09:05.581316 +2025-01-06 22:09:05,582 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:09:05,597 INFO WriterThread:2121135 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/run-5jdyg1q2.wandb +2025-01-06 22:09:05,597 DEBUG SenderThread:2121135 [sender.py:send():379] send: header +2025-01-06 22:09:05,604 DEBUG SenderThread:2121135 [sender.py:send():379] send: run +2025-01-06 22:09:05,839 INFO SenderThread:2121135 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files +2025-01-06 22:09:05,839 INFO SenderThread:2121135 [sender.py:_start_run_threads():1188] run started: 5jdyg1q2 with start time 1736219345.581746 +2025-01-06 22:09:05,857 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:09:05,858 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:09:05,903 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:09:05,908 DEBUG HandlerThread:2121135 [system_info.py:__init__():26] System info init +2025-01-06 22:09:05,908 DEBUG HandlerThread:2121135 [system_info.py:__init__():41] System info init done +2025-01-06 22:09:05,908 INFO HandlerThread:2121135 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:09:05,908 INFO SystemMonitor:2121135 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:09:05,909 INFO HandlerThread:2121135 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:09:05,909 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:09:05,910 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:09:05,910 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:09:05,911 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:09:05,911 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:09:05,922 DEBUG HandlerThread:2121135 [system_info.py:probe():152] Probing system +2025-01-06 22:09:05,923 DEBUG HandlerThread:2121135 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:09:05,928 DEBUG HandlerThread:2121135 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:09:05,928 DEBUG HandlerThread:2121135 [system_info.py:probe():200] Probing system done +2025-01-06 22:09:05,928 DEBUG HandlerThread:2121135 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:09:05.922215', 'startedAt': '2025-01-07T03:09:05.573559', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.184875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4493.53, 'min': 800.0, 'max': 4700.0}, {'current': 4485.963, 'min': 800.0, 'max': 4700.0}, {'current': 4492.566, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.189, 'min': 800.0, 'max': 4700.0}, {'current': 4499.977, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 551.6594390869141}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:09:05,928 INFO HandlerThread:2121135 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:09:05,928 INFO HandlerThread:2121135 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:09:05,928 DEBUG HandlerThread:2121135 [system_info.py:_save_code():44] Saving code +2025-01-06 22:09:05,930 DEBUG HandlerThread:2121135 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:09:05,930 DEBUG HandlerThread:2121135 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:09:05,993 DEBUG HandlerThread:2121135 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:09:05,993 INFO HandlerThread:2121135 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:09:05,998 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:09:05,998 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:09:05,999 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:09:05,999 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:09:06,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:09:06,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:09:06,244 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:09:06,244 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:09:06,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:06,246 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/5pmvsrf8-wandb-metadata.json +2025-01-06 22:09:06,247 INFO HandlerThread:2121135 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/tb" + save: true +} +control { + relay_id: "50707d9cb10d47b0b7fbc7b5efa7cbb4" +} +_info { + stream_id: "5jdyg1q2" +} + +2025-01-06 22:09:06,247 DEBUG HandlerThread:2121135 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:09:06,271 INFO wandb-upload_2:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/8dgsibm9-diff.patch +2025-01-06 22:09:06,272 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/lus1ltsz-code/FISH/eval_robot.py +2025-01-06 22:09:06,378 DEBUG SenderThread:2121135 [sender.py:send():379] send: telemetry +2025-01-06 22:09:06,378 DEBUG SenderThread:2121135 [sender.py:send():379] send: tbrecord +2025-01-06 22:09:06,378 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:09:06,379 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 with policy live +2025-01-06 22:09:06,574 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/j288eula-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code/FISH/eval_robot.py +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/diff.patch +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-metadata.json +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/requirements.txt +2025-01-06 22:09:06,842 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:06,842 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code +2025-01-06 22:09:06,842 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code/FISH +2025-01-06 22:09:07,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:08,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:08,840 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:09,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:10,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:11,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:11,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:12,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:13,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:14,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:15,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:16,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:16,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:17,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:18,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:19,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:20,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:21,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:09:21,244 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:09:21,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:21,379 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:21,433 DEBUG SenderThread:2121135 [sender.py:send():379] send: config +2025-01-06 22:09:22,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:22,844 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:23,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:24,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:24,844 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:25,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:26,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:26,964 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:27,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:28,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:28,846 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:29,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:30,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:30,846 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:31,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:32,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:32,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:32,847 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:33,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:34,249 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:34,847 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:35,249 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:36,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:09:36,244 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:09:36,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:36,848 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:37,253 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:37,384 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:37,848 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml +2025-01-06 22:09:38,253 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:39,253 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:40,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:41,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:42,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:43,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:43,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:44,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:45,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:46,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:47,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:48,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:48,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:49,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:50,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:51,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:09:51,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:09:51,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:52,256 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:53,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:53,401 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:54,256 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:55,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:56,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:57,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:58,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:59,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:59,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:00,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:01,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:02,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:02,907 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:10:02,907 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 with policy now +2025-01-06 22:10:02,908 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:10:02,908 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:10:02,909 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:10:02,910 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:10:03,130 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/y0dbacm7-media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 +2025-01-06 22:10:03,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:03,855 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 +2025-01-06 22:10:03,855 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:10:03,856 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos +2025-01-06 22:10:03,856 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:10:03,856 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media +2025-01-06 22:10:04,129 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/scjzulj6-media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 +2025-01-06 22:10:04,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:04,856 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:04,917 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:05,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:05,912 DEBUG SystemMonitor:2121135 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:10:05,912 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:10:06,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:10:06,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:10:06,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:07,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:08,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:08,857 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:09,269 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:10,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:10,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:10,858 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:10,858 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:10:11,104 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/vh9otb7u-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:10:11,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:12,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:12,859 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:13,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:14,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:15,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:15,707 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:16,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:16,860 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:17,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:18,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:19,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:20,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:21,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:10:21,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:10:21,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:21,391 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:22,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:23,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:24,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:25,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:26,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:27,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:27,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:28,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:29,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:30,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:31,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:32,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:32,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:33,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:34,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:35,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:35,913 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:10:36,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:10:36,246 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:10:36,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:37,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:37,381 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:38,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:39,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:40,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:41,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:42,265 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:42,467 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:42,509 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:10:42,509 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:10:42,509 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 with policy now +2025-01-06 22:10:42,516 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:10:42,517 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:10:42,517 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:10:42,724 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/sjlnaztj-media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 +2025-01-06 22:10:42,868 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:42,868 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:10:42,868 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 +2025-01-06 22:10:42,868 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:10:43,145 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/iic8u53i-media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 +2025-01-06 22:10:43,269 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:44,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:44,869 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:45,265 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:46,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:47,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:48,252 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:48,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:48,870 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:49,268 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:50,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:50,871 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:51,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:10:51,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:10:51,269 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:51,640 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:10:51,640 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:10:51,650 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:10:51,871 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:10:51,871 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:10:52,269 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:52,872 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:53,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:54,057 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:54,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:54,872 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:55,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:56,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:56,873 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:57,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:58,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:58,874 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:59,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:59,957 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:00,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:01,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:02,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:03,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:04,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:05,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:05,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:05,913 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:11:06,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:11:06,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:11:06,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:07,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:08,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:09,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:10,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:10,359 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:11,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:12,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:13,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:14,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:15,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:16,249 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:16,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:17,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:18,274 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:19,274 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:20,274 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:21,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:11:21,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:11:21,293 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:21,333 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:22,276 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:22,883 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:23,143 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:11:23,143 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 with policy now +2025-01-06 22:11:23,146 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:11:23,147 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:11:23,147 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:11:23,147 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:11:23,275 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:23,405 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/uil60335-media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 +2025-01-06 22:11:23,883 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:11:23,883 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 +2025-01-06 22:11:23,884 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:11:24,147 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/fkbgq9uw-media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 +2025-01-06 22:11:24,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:24,884 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:25,283 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:25,884 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:26,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:26,763 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:27,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:28,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:28,885 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:29,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:30,279 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:30,886 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:30,886 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:11:31,130 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/y162t2gw-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:11:31,279 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:31,469 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:11:31,469 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:11:31,469 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:11:31,781 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:31,886 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:11:32,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:32,887 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:33,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:34,284 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:35,279 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:35,914 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:11:36,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:11:36,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:11:36,293 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:36,888 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:37,279 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:37,305 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:38,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:39,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:40,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:41,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:42,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:43,251 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:43,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:44,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:45,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:46,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:47,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:48,251 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:48,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:49,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:50,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:51,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:11:51,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:11:51,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:52,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:53,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:53,404 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:54,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:55,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:56,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:57,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:58,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:59,252 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:59,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:00,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:01,284 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:02,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:02,872 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:12:02,872 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 with policy now +2025-01-06 22:12:02,877 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:12:02,877 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:02,877 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:02,878 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:02,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:02,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:02,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 +2025-01-06 22:12:02,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:12:03,083 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/hvp5dubi-media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 +2025-01-06 22:12:03,284 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:03,509 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/3mi00jkn-media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 +2025-01-06 22:12:03,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:04,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:04,897 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:04,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:05,286 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:05,898 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:05,914 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:12:06,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:12:06,248 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:12:06,293 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:07,291 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:08,294 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:08,899 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:09,301 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:10,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:10,361 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:10,900 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:11,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:11,521 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:11,521 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:11,522 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:11,900 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:12:11,900 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:12,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:12,900 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:13,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:14,307 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:14,901 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:15,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:15,364 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:16,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:16,902 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:17,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:18,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:19,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:20,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:21,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:12:21,246 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:12:21,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:21,372 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:22,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:23,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:24,307 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:25,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:26,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:27,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:27,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:28,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:29,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:30,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:31,310 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:32,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:32,310 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:33,310 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:34,315 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:35,316 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:35,914 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:12:36,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:12:36,246 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:12:36,317 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:37,317 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:37,360 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:38,318 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:39,317 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:40,317 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:41,318 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:42,318 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:42,394 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:43,320 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:43,633 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:12:43,633 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 with policy now +2025-01-06 22:12:43,637 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:12:43,637 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:43,637 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:43,639 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:43,852 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/h0fpj5v1-media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 +2025-01-06 22:12:43,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:43,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 +2025-01-06 22:12:43,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:12:44,166 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/oiwywnv6-media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 +2025-01-06 22:12:44,335 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:44,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:45,329 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:45,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:46,324 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:47,339 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:48,325 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:48,338 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:48,912 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:49,330 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:50,338 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:51,251 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:12:51,251 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:12:51,343 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:52,336 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:52,576 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:52,576 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:52,577 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:52,750 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:12:52,750 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:12:52,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:52,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:12:52,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:52,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 22:12:52,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media +2025-01-06 22:12:52,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval +2025-01-06 22:12:52,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table +2025-01-06 22:12:52,974 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:12:53,086 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/1ktdx099-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 22:12:53,141 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/bbvw75fu-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:12:53,210 INFO wandb-upload_2:2121135 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyy4pbl06 +2025-01-06 22:12:53,334 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:53,341 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:12:53,381 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:12:53,623 INFO SenderThread:2121135 [sender.py:send_request_log_artifact():1525] logged artifact run-5jdyg1q2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU0ODAxMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDczMDg2Nw==', 'latestArtifact': None}} +2025-01-06 22:12:53,623 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:12:53,623 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 22:12:53,623 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:53,623 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,624 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:53,624 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,624 DEBUG SenderThread:2121135 [sender.py:send():379] send: telemetry +2025-01-06 22:12:53,624 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,625 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,625 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,625 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,625 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,626 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,626 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,628 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,628 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,628 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,628 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,629 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,629 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,629 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,629 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,630 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,630 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,630 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,630 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,631 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,631 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,631 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,631 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:12:53,854 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/hrkyf06d-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 22:12:53,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:12:53,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:53,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json +2025-01-06 22:12:53,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval +2025-01-06 22:12:53,985 INFO wandb-upload_2:2121135 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpivtif3jc +2025-01-06 22:12:54,095 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/ayx5ya39-media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json +2025-01-06 22:12:54,334 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:54,408 INFO SenderThread:2121135 [sender.py:send_request_log_artifact():1525] logged artifact run-5jdyg1q2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU0ODAyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDczMDg2OQ==', 'latestArtifact': None}} +2025-01-06 22:12:54,408 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:12:54,408 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json with policy now +2025-01-06 22:12:54,408 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:54,409 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:54,409 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:54,409 DEBUG SenderThread:2121135 [sender.py:send():379] send: exit +2025-01-06 22:12:54,409 INFO SenderThread:2121135 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:12:54,409 INFO SenderThread:2121135 [sender.py:send_exit():588] handling runtime: 228 +2025-01-06 22:12:54,410 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:54,410 INFO SenderThread:2121135 [sender.py:send_exit():594] send defer +2025-01-06 22:12:54,410 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,411 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:12:54,412 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:12:54,412 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:12:54,412 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:12:54,412 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,412 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:12:54,412 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:12:54,412 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:12:54,412 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:12:54,412 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,412 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:12:54,412 INFO HandlerThread:2121135 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:12:54,413 DEBUG SystemMonitor:2121135 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:12:54,413 DEBUG SystemMonitor:2121135 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:12:54,413 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:12:54,413 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:12:54,464 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:12:54,464 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:12:54,464 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:12:54,465 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:12:54,465 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:12:54,465 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:12:54,465 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:12:54,465 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,465 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:12:54,465 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:12:54,465 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:12:54,465 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:12:54,466 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,466 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:12:54,679 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/ygo9w8k1-media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json +2025-01-06 22:12:54,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:54,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:56,916 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:13:00,582 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:00,582 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:13:00,582 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:13:00,583 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:00,583 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:13:00,583 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:13:00,584 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:00,584 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:13:00,584 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:13:00,584 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:00,584 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:13:00,585 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:13:00,585 DEBUG SenderThread:2121135 [sender.py:send():379] send: summary +2025-01-06 22:13:00,589 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:13:00,589 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:00,589 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:13:00,589 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:13:00,589 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:00,589 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:13:00,589 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:00,589 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:13:00,593 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:13:00,727 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:13:00,728 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:00,728 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:13:00,728 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:00,728 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:13:00,917 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml +2025-01-06 22:13:00,918 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:13:01,391 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:03,149 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:13:03,149 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:03,149 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:03,150 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:13:03,150 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:03,150 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:13:03,150 INFO SenderThread:2121135 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:13:03,152 INFO SenderThread:2121135 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:13:03,154 INFO SenderThread:2121135 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:13:03,165 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:13:03,165 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:03,165 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:13:03,165 DEBUG SenderThread:2121135 [sender.py:send():379] send: artifact +2025-01-06 22:13:03,391 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:03,534 INFO SenderThread:2121135 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 22:13:03,534 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:03,534 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:13:03,534 INFO SenderThread:2121135 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:13:03,918 INFO SenderThread:2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml config.yaml +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json wandb-summary.json +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/requirements.txt requirements.txt +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:13:03,921 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log output.log +2025-01-06 22:13:03,921 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/diff.patch diff.patch +2025-01-06 22:13:03,924 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:13:03,924 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:13:03,926 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:03,928 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:03,928 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:13:03,928 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:03,928 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:13:03,928 INFO SenderThread:2121135 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:13:04,100 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml +2025-01-06 22:13:04,109 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:13:04,137 INFO wandb-upload_2:2121135 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/requirements.txt +2025-01-06 22:13:04,199 INFO wandb-upload_3:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/up8yxneq-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:13:04,201 INFO wandb-upload_4:2121135 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:13:04,392 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:04,392 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:04,401 INFO Thread-11 (_thread_body):2121135 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:13:04,402 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:04,402 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:13:04,402 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:04,402 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:13:04,402 INFO SenderThread:2121135 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:13:04,405 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:13:04,405 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:04,405 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:13:04,405 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:04,405 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:13:04,405 INFO SenderThread:2121135 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:13:04,497 INFO SenderThread:2121135 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:13:04,497 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:13:04,497 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:04,497 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:13:04,497 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:04,497 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:13:04,498 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:13:04,498 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:04,498 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:13:04,498 DEBUG SenderThread:2121135 [sender.py:send():379] send: final +2025-01-06 22:13:04,498 DEBUG SenderThread:2121135 [sender.py:send():379] send: footer +2025-01-06 22:13:04,498 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:04,498 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:13:04,499 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:04,499 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:04,499 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:13:04,499 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:13:04,500 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:04,503 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:13:04,503 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:13:04,504 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:13:04,556 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:04,557 INFO MainThread:2121135 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:13:04,557 INFO MainThread:2121135 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:13:04,557 INFO MainThread:2121135 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:13:04,557 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:13:04,557 INFO HandlerThread:2121135 [handler.py:finish():882] shutting down handler +2025-01-06 22:13:05,503 INFO WriterThread:2121135 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/run-5jdyg1q2.wandb +2025-01-06 22:13:05,556 INFO SenderThread:2121135 [sender.py:finish():1615] shutting down sender +2025-01-06 22:13:05,557 INFO SenderThread:2121135 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:13:05,557 INFO SenderThread:2121135 [file_pusher.py:join():175] waiting for file pusher diff --git a/5jdyg1q2/wandb/debug.log b/5jdyg1q2/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b740ba16729905aa26e63a2a899084720d410f5d --- /dev/null +++ b/5jdyg1q2/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Configure stats pid to 2120949 +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/settings +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug.log +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug-internal.log +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2'} +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:init():619] starting backend +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:init():623] setting up manager +2025-01-06 22:09:05,580 INFO MainThread:2120949 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:09:05,581 INFO MainThread:2120949 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:09:05,596 INFO MainThread:2120949 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:09:05,602 INFO MainThread:2120949 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:09:05,857 INFO MainThread:2120949 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:09:05,899 INFO MainThread:2120949 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:09:05,899 INFO MainThread:2120949 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:09:06,244 INFO MainThread:2120949 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:09:06,244 INFO MainThread:2120949 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:09:06,244 INFO MainThread:2120949 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:09:06,245 INFO MainThread:2120949 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:09:06,246 INFO MainThread:2120949 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:09:06,246 INFO MainThread:2120949 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/tb, True +2025-01-06 22:09:21,433 INFO MainThread:2120949 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:13:05,558 WARNING MsgRouterThr:2120949 [router.py:message_loop():77] message_loop has been closed diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code/FISH/eval_robot.py b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01c0d3e64d88fa2ce6744a0fb63c6d3c70cf8140 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1009_1' +wandb_notes: + desc: null + value: 1009_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736219345 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/diff.patch b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..43dd1a505c5d25a547a09b1d8e6dc31eb2a6a985 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..101f9bd 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..f90fdbaec1d21eb65c7684fdad66f28da3d0a5b8 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 25, 1.0, 0.9956949352226687, 1, 0.8742487429824013, 1, 0.8909462019019245, 1, 1, 1]]} \ No newline at end of file diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7810ac4a4e95bcf1f6165bed0226d89a697d730e --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6426b8e8aadca81f0c9e61bd8f64f1578e6f4f97caef958186eed7e5f2d31306 +size 1145928 diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b1d941e06fd48491d4f5b85e3fec1eeadf40e7ff --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6e43c5f90cc9d422ca7e5b4677d6816f44b66e7a4a4594bffe010680ec0994 +size 1146021 diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df52b46c8bd1283cb1de1c8792d267619d8ecd32 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edc3f417d98fcdf1019e50974244ca5e7dc844b3fb6a9fcbfe608cb43e538275 +size 1139041 diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba757932e4150f86d52a9606db762954ae10c91e --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc101df27963f59b97f2dec5c530041fcb86770368f574b840b6d71511b01799 +size 1141420 diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..791d1c5fff05fe90f5d22e5b7ec81991cf2b1b13 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9c235f657b8c4c13ec23d5ad66221e2a5888753182f820a13a8677a3f3ed0cf +size 1138718 diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a8697260be2cd76df64c2022cd00a311277f82f5 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log @@ -0,0 +1,278 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736219361.433917]: resetting environment +[INFO] [1736219361.436854]: cleared current plan +[INFO] [1736219361.438741]: moving to home +[INFO] [1736219363.244087]: reached home +[INFO] [1736219363.245909]: reset action history +[INFO] [1736219365.954636]: environment reset +Starting episode 0 +[INFO] [1736219365.957198]: resetting environment +[INFO] [1736219365.960351]: cleared current plan +[INFO] [1736219365.961171]: moving to home +[INFO] [1736219366.963463]: reached home +[INFO] [1736219366.964930]: reset action history +[INFO] [1736219369.672939]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736219371.241243]: resetting environment +[INFO] [1736219371.244173]: cleared current plan +[INFO] [1736219371.245346]: moving to home +[INFO] [1736219372.247900]: reached home +[INFO] [1736219372.249430]: reset action history +[INFO] [1736219374.957147]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/episode_rosbags/episode_0_2025-01-06-22-09-34.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2120949.log +started roslaunch server http://158.130.50.37:35391/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2121240] +process[print_text-2]: started with pid [2121264] +process[rosbag_record-3]: started with pid [2121265] +started bagging! +[INFO] [1736219401.941373]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736219402.688227]: Stored episode 1. +[INFO] [1736219402.688435]: Saving video... +[INFO] [1736219402.895885]: Video saved! +Starting episode 1 +[INFO] [1736219402.908785]: resetting environment +[INFO] [1736219402.914438]: cleared current plan +[INFO] [1736219402.915388]: moving to home +[INFO] [1736219406.518850]: reached home +[INFO] [1736219406.520180]: reset action history +[INFO] [1736219409.227193]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736219410.716643]: resetting environment +[INFO] [1736219410.718607]: cleared current plan +[INFO] [1736219410.718794]: moving to home +[INFO] [1736219411.720237]: reached home +[INFO] [1736219411.720656]: reset action history +[INFO] [1736219414.476131]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/episode_rosbags/episode_1_2025-01-06-22-10-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2120949.log +started roslaunch server http://158.130.50.37:43425/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2121375] +process[print_text-5]: started with pid [2121376] +process[rosbag_record-6]: started with pid [2121377] +started bagging! +[INFO] [1736219441.409036]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736219442.220037]: Stored episode 2. +[INFO] [1736219442.220859]: Saving video... +[INFO] [1736219442.466439]: Video saved! +Starting episode 2 +[INFO] [1736219442.491667]: resetting environment +[INFO] [1736219442.503482]: cleared current plan +[INFO] [1736219442.505358]: moving to home +[INFO] [1736219446.112765]: reached home +[INFO] [1736219446.115022]: reset action history +[INFO] [1736219448.841483]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736219450.996491]: resetting environment +[INFO] [1736219451.006784]: cleared current plan +[INFO] [1736219451.007514]: moving to home +[INFO] [1736219452.009042]: reached home +[INFO] [1736219452.009884]: reset action history +[INFO] [1736219454.717027]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/episode_rosbags/episode_2_2025-01-06-22-10-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2120949.log +started roslaunch server http://158.130.50.37:36021/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2121511] +process[print_text-8]: started with pid [2121512] +process[rosbag_record-9]: started with pid [2121513] +started bagging! +For topic gripper_width: timestamp difference is -122347282 for nearest: 1736219457690742566 - target: 1736219457813089848 at idx 41 +[INFO] [1736219481.685632]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736219482.485070]: Stored episode 3. +[INFO] [1736219482.486478]: Saving video... +[INFO] [1736219483.106163]: Video saved! +Starting episode 3 +[INFO] [1736219483.149113]: resetting environment +[INFO] [1736219483.157286]: cleared current plan +[INFO] [1736219483.157653]: moving to home +[WARN] [1736219483.058576]: Plan exhausted +[WARN] [1736219483.098530]: Plan exhausted +[WARN] [1736219483.138622]: Plan exhausted +[INFO] [1736219486.763151]: reached home +[INFO] [1736219486.767221]: reset action history +[INFO] [1736219489.544384]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736219490.719174]: resetting environment +[INFO] [1736219490.733410]: cleared current plan +[INFO] [1736219490.734928]: moving to home +[INFO] [1736219491.739473]: reached home +[INFO] [1736219491.740827]: reset action history +[INFO] [1736219494.471457]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/episode_rosbags/episode_3_2025-01-06-22-11-34.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2120949.log +started roslaunch server http://158.130.50.37:46723/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2121646] +process[print_text-11]: started with pid [2121647] +process[rosbag_record-12]: started with pid [2121648] +started bagging! +[INFO] [1736219521.424362]: Storing episode... +[print_text-11] killing on exit +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[WARN] [1736219522.580868]: Plan exhausted +[INFO] [1736219522.230133]: Stored episode 4. +[INFO] [1736219522.234095]: Saving video... +[INFO] [1736219522.831908]: Video saved! +Starting episode 4 +[INFO] [1736219522.886228]: resetting environment +[INFO] [1736219522.896183]: cleared current plan +[INFO] [1736219522.896485]: moving to home +[WARN] [1736219522.619464]: Plan exhausted +[WARN] [1736219522.658846]: Plan exhausted +[WARN] [1736219522.699316]: Plan exhausted +[WARN] [1736219522.738924]: Plan exhausted +[WARN] [1736219522.788468]: Plan exhausted +[WARN] [1736219522.819328]: Plan exhausted +[WARN] [1736219522.859492]: Plan exhausted +[INFO] [1736219526.500096]: reached home +[INFO] [1736219526.510645]: reset action history +[INFO] [1736219529.315801]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736219531.542441]: resetting environment +[INFO] [1736219531.561197]: cleared current plan +[INFO] [1736219531.567921]: moving to home +[INFO] [1736219532.582961]: reached home +[INFO] [1736219532.585842]: reset action history +[INFO] [1736219535.363321]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/episode_rosbags/episode_4_2025-01-06-22-12-15.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2120949.log +started roslaunch server http://158.130.50.37:45493/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2121782] +process[print_text-14]: started with pid [2121783] +process[rosbag_record-15]: started with pid [2121807] +started bagging! +[INFO] [1736219562.335830]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736219563.118133]: Stored episode 5. +[INFO] [1736219563.131694]: Saving video... +[INFO] [1736219563.595281]: Video saved! +[INFO] [1736219563.638303]: resetting environment +[INFO] [1736219563.657400]: cleared current plan +[INFO] [1736219563.663685]: moving to home +[WARN] [1736219563.498558]: Plan exhausted +[WARN] [1736219563.542307]: Plan exhausted +[WARN] [1736219563.578531]: Plan exhausted +[WARN] [1736219563.618522]: Plan exhausted +[INFO] [1736219567.277365]: reached home +[INFO] [1736219567.280416]: reset action history +[INFO] [1736219570.228957]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/requirements.txt b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-metadata.json b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0d38e0c2ee54ff199a249386cfb620a2757c7b4a --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T03:09:05.922215", + "startedAt": "2025-01-07T03:09:05.573559", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.184875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4493.53, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4485.963, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4492.566, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.004, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.189, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.977, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 551.6594390869141 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ba8ad3c0c3ebd10c7d7327ba2fcdd229951de2f4 --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "6426b8e8aadca81f0c9e61bd8f64f1578e6f4f97caef958186eed7e5f2d31306", "size": 1145928, "path": "media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4"}, "global_step": 662, "_timestamp": 1736219573.2627552, "_runtime": 227.68100905418396, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "3f6e43c5f90cc9d422ca7e5b4677d6816f44b66e7a4a4594bffe010680ec0994", "size": 1146021, "path": "media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4"}, "eval/num_success": 5.0, "episode": 25.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "edc3f417d98fcdf1019e50974244ca5e7dc844b3fb6a9fcbfe608cb43e538275", "size": 1139041, "path": "media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "dc101df27963f59b97f2dec5c530041fcb86770368f574b840b6d71511b01799", "size": 1141420, "path": "media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d9c235f657b8c4c13ec23d5ad66221e2a5888753182f820a13a8677a3f3ed0cf", "size": 1138718, "path": "media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://hn3s7iquu5mstm7g9v95hig7s0lk8otmn57bivwkn8kzxk8od41r5ltt449srst2hm407jt6em0kofeqii1evu9s9e1pp06c3ai3kpug8j865wuwkut3jzz2zu507vxg/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://2g2qrudwsa163ed1j0u637bh0ku9rvb6l511qy2gaii00lab6hpsfrys7w55988yah4y3pw3n7kz4uxcy3g1sn92fl33yajpvqhn2z3e1g6dbvodu8jtgfy3wgapwtlt:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 25, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9956949352226687, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.8742487429824013, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.8909462019019245, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "bb805e4eaba65ae7d98f67e51941c72e8ab0429cc1b1f6943f66bedfeb96deef", "size": 420, "artifact_path": "wandb-client-artifact://4j8ssln3iyp9fl6g5a0beyd5p7dj7dn5rekb6aivat6hfwwumecvge6ka2ketavkw1y0p7ryxio9drb9ks5lhumvna2l00vyoeh2adxs6lbavrrarr7xrlgin2gnxxod/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://wczdgzutw42f0c54tvudy67974sbg4guydbi9wveygp9qrp47zwrn9mu32cwicw8uwefjmm39e3apj66ed3ffc6coyz9663dj52as4q8qrxoowaomjik673ivulllv5q:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 228}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9956949353218079, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.874248743057251, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.8909462094306946, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug-internal.log b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1250fecfa6c3bafe0633cfd790dc2d07192e710f --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug-internal.log @@ -0,0 +1,716 @@ +2025-01-06 22:09:05,581 INFO StreamThr :2121135 [internal.py:wandb_internal():85] W&B internal server running at pid: 2121135, started at: 2025-01-06 22:09:05.581316 +2025-01-06 22:09:05,582 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:09:05,597 INFO WriterThread:2121135 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/run-5jdyg1q2.wandb +2025-01-06 22:09:05,597 DEBUG SenderThread:2121135 [sender.py:send():379] send: header +2025-01-06 22:09:05,604 DEBUG SenderThread:2121135 [sender.py:send():379] send: run +2025-01-06 22:09:05,839 INFO SenderThread:2121135 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files +2025-01-06 22:09:05,839 INFO SenderThread:2121135 [sender.py:_start_run_threads():1188] run started: 5jdyg1q2 with start time 1736219345.581746 +2025-01-06 22:09:05,857 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:09:05,858 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:09:05,903 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:09:05,908 DEBUG HandlerThread:2121135 [system_info.py:__init__():26] System info init +2025-01-06 22:09:05,908 DEBUG HandlerThread:2121135 [system_info.py:__init__():41] System info init done +2025-01-06 22:09:05,908 INFO HandlerThread:2121135 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:09:05,908 INFO SystemMonitor:2121135 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:09:05,909 INFO HandlerThread:2121135 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:09:05,909 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:09:05,910 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:09:05,910 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:09:05,911 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:09:05,911 INFO SystemMonitor:2121135 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:09:05,922 DEBUG HandlerThread:2121135 [system_info.py:probe():152] Probing system +2025-01-06 22:09:05,923 DEBUG HandlerThread:2121135 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:09:05,928 DEBUG HandlerThread:2121135 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:09:05,928 DEBUG HandlerThread:2121135 [system_info.py:probe():200] Probing system done +2025-01-06 22:09:05,928 DEBUG HandlerThread:2121135 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:09:05.922215', 'startedAt': '2025-01-07T03:09:05.573559', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.184875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4493.53, 'min': 800.0, 'max': 4700.0}, {'current': 4485.963, 'min': 800.0, 'max': 4700.0}, {'current': 4492.566, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.189, 'min': 800.0, 'max': 4700.0}, {'current': 4499.977, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 551.6594390869141}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:09:05,928 INFO HandlerThread:2121135 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:09:05,928 INFO HandlerThread:2121135 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:09:05,928 DEBUG HandlerThread:2121135 [system_info.py:_save_code():44] Saving code +2025-01-06 22:09:05,930 DEBUG HandlerThread:2121135 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:09:05,930 DEBUG HandlerThread:2121135 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:09:05,993 DEBUG HandlerThread:2121135 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:09:05,993 INFO HandlerThread:2121135 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:09:05,998 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:09:05,998 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:09:05,999 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:09:05,999 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:09:06,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:09:06,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:09:06,244 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:09:06,244 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:09:06,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:06,246 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/5pmvsrf8-wandb-metadata.json +2025-01-06 22:09:06,247 INFO HandlerThread:2121135 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/tb" + save: true +} +control { + relay_id: "50707d9cb10d47b0b7fbc7b5efa7cbb4" +} +_info { + stream_id: "5jdyg1q2" +} + +2025-01-06 22:09:06,247 DEBUG HandlerThread:2121135 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:09:06,271 INFO wandb-upload_2:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/8dgsibm9-diff.patch +2025-01-06 22:09:06,272 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/lus1ltsz-code/FISH/eval_robot.py +2025-01-06 22:09:06,378 DEBUG SenderThread:2121135 [sender.py:send():379] send: telemetry +2025-01-06 22:09:06,378 DEBUG SenderThread:2121135 [sender.py:send():379] send: tbrecord +2025-01-06 22:09:06,378 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:09:06,379 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 with policy live +2025-01-06 22:09:06,574 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/j288eula-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code/FISH/eval_robot.py +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/diff.patch +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-metadata.json +2025-01-06 22:09:06,841 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/requirements.txt +2025-01-06 22:09:06,842 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:06,842 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code +2025-01-06 22:09:06,842 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code/FISH +2025-01-06 22:09:07,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:08,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:08,840 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:09,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:10,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:11,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:11,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:12,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:13,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:14,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:15,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:16,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:16,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:17,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:18,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:19,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:20,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:21,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:09:21,244 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:09:21,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:21,379 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:21,433 DEBUG SenderThread:2121135 [sender.py:send():379] send: config +2025-01-06 22:09:22,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:22,844 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:23,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:24,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:24,844 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:25,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:26,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:26,964 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:27,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:28,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:28,846 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:29,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:30,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:30,846 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:31,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:32,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:32,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:32,847 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:33,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:34,249 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:34,847 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:35,249 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:36,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:09:36,244 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:09:36,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:36,848 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:09:37,253 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:37,384 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:37,848 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml +2025-01-06 22:09:38,253 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:39,253 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:40,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:41,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:42,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:43,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:43,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:44,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:45,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:46,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:47,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:48,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:48,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:49,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:50,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:51,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:09:51,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:09:51,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:52,256 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:53,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:53,401 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:54,256 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:55,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:56,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:57,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:58,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:09:59,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:09:59,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:00,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:01,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:02,257 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:02,907 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:10:02,907 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 with policy now +2025-01-06 22:10:02,908 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:10:02,908 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:10:02,909 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:10:02,910 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:10:03,130 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/y0dbacm7-media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 +2025-01-06 22:10:03,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:03,855 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 +2025-01-06 22:10:03,855 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:10:03,856 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos +2025-01-06 22:10:03,856 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:10:03,856 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media +2025-01-06 22:10:04,129 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/scjzulj6-media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 +2025-01-06 22:10:04,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:04,856 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:04,917 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:05,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:05,912 DEBUG SystemMonitor:2121135 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:10:05,912 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:10:06,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:10:06,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:10:06,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:07,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:08,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:08,857 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:09,269 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:10,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:10,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:10,858 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:10,858 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:10:11,104 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/vh9otb7u-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:10:11,258 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:12,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:12,859 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:13,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:14,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:15,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:15,707 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:16,259 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:16,860 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:17,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:18,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:19,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:20,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:21,244 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:10:21,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:10:21,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:21,391 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:22,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:23,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:24,261 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:25,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:26,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:27,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:27,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:28,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:29,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:30,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:31,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:32,247 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:32,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:33,262 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:34,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:35,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:35,913 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:10:36,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:10:36,246 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:10:36,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:37,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:37,381 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:38,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:39,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:40,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:41,263 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:42,265 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:42,467 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:42,509 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:10:42,509 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:10:42,509 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 with policy now +2025-01-06 22:10:42,516 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:10:42,517 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:10:42,517 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:10:42,724 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/sjlnaztj-media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 +2025-01-06 22:10:42,868 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:42,868 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:10:42,868 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 +2025-01-06 22:10:42,868 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:10:43,145 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/iic8u53i-media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 +2025-01-06 22:10:43,269 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:44,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:44,869 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:45,265 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:46,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:47,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:48,252 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:48,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:48,870 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:49,268 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:50,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:50,871 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:51,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:10:51,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:10:51,269 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:51,640 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:10:51,640 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:10:51,650 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:10:51,871 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:10:51,871 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:10:52,269 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:52,872 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:53,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:54,057 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:10:54,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:54,872 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:55,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:56,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:56,873 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:57,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:58,270 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:58,874 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:10:59,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:10:59,957 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:00,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:01,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:02,271 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:03,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:04,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:05,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:05,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:05,913 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:11:06,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:11:06,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:11:06,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:07,272 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:08,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:09,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:10,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:10,359 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:11,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:12,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:13,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:14,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:15,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:16,249 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:16,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:17,273 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:18,274 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:19,274 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:20,274 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:21,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:11:21,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:11:21,293 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:21,333 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:22,276 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:22,883 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:23,143 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:11:23,143 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 with policy now +2025-01-06 22:11:23,146 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:11:23,147 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:11:23,147 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:11:23,147 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:11:23,275 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:23,405 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/uil60335-media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 +2025-01-06 22:11:23,883 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:11:23,883 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 +2025-01-06 22:11:23,884 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:11:24,147 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/fkbgq9uw-media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 +2025-01-06 22:11:24,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:24,884 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:25,283 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:25,884 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:26,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:26,763 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:27,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:28,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:28,885 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:29,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:30,279 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:30,886 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:30,886 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:11:31,130 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/y162t2gw-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:11:31,279 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:31,469 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:11:31,469 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:11:31,469 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:11:31,781 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:31,886 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:11:32,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:32,887 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:33,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:34,284 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:35,279 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:35,914 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:11:36,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:11:36,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:11:36,293 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:36,888 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:11:37,279 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:37,305 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:38,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:39,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:40,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:41,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:42,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:43,251 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:43,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:44,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:45,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:46,280 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:47,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:48,251 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:48,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:49,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:50,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:51,245 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:11:51,245 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:11:51,289 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:52,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:53,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:53,404 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:54,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:55,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:56,281 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:57,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:58,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:11:59,252 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:11:59,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:00,282 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:01,284 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:02,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:02,872 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:12:02,872 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 with policy now +2025-01-06 22:12:02,877 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:12:02,877 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:02,877 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:02,878 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:02,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:02,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:02,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 +2025-01-06 22:12:02,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:12:03,083 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/hvp5dubi-media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 +2025-01-06 22:12:03,284 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:03,509 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/3mi00jkn-media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 +2025-01-06 22:12:03,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:04,285 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:04,897 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:04,897 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:05,286 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:05,898 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:05,914 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:12:06,248 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:12:06,248 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:12:06,293 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:07,291 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:08,294 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:08,899 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:09,301 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:10,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:10,361 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:10,900 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:11,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:11,521 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:11,521 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:11,522 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:11,900 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:12:11,900 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:12,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:12,900 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:13,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:14,307 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:14,901 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:15,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:15,364 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:16,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:16,902 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:17,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:18,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:19,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:20,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:21,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:12:21,246 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:12:21,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:21,372 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:22,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:23,306 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:24,307 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:25,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:26,308 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:27,254 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:27,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:28,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:29,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:30,309 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:31,310 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:32,255 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:32,310 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:33,310 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:34,315 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:35,316 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:35,914 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:12:36,246 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:12:36,246 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:12:36,317 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:37,317 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:37,360 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:38,318 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:39,317 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:40,317 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:41,318 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:42,318 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:42,394 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:43,320 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:43,633 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:12:43,633 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 with policy now +2025-01-06 22:12:43,637 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:12:43,637 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:43,637 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:43,639 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:43,852 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/h0fpj5v1-media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 +2025-01-06 22:12:43,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:43,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 +2025-01-06 22:12:43,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval +2025-01-06 22:12:44,166 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/oiwywnv6-media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 +2025-01-06 22:12:44,335 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:44,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:45,329 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:45,911 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:46,324 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:47,339 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:48,325 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:48,338 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:48,912 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:49,330 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:50,338 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:51,251 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:12:51,251 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:12:51,343 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:52,336 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:52,576 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:52,576 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:52,577 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:52,750 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:12:52,750 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:12:52,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:52,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:12:52,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:52,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 22:12:52,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media +2025-01-06 22:12:52,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval +2025-01-06 22:12:52,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table +2025-01-06 22:12:52,974 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:12:53,086 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/1ktdx099-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 22:12:53,141 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/bbvw75fu-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:12:53,210 INFO wandb-upload_2:2121135 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyy4pbl06 +2025-01-06 22:12:53,334 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:53,341 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:12:53,381 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:12:53,623 INFO SenderThread:2121135 [sender.py:send_request_log_artifact():1525] logged artifact run-5jdyg1q2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU0ODAxMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDczMDg2Nw==', 'latestArtifact': None}} +2025-01-06 22:12:53,623 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:12:53,623 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-06 22:12:53,623 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:53,623 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,624 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:12:53,624 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,624 DEBUG SenderThread:2121135 [sender.py:send():379] send: telemetry +2025-01-06 22:12:53,624 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,625 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,625 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,625 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,625 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,626 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,626 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,628 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,628 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,628 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,628 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,629 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,629 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,629 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,629 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,630 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,630 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,630 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,630 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,631 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,631 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:53,631 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:53,631 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:12:53,854 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/hrkyf06d-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 22:12:53,914 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:12:53,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:53,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json +2025-01-06 22:12:53,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval +2025-01-06 22:12:53,985 INFO wandb-upload_2:2121135 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpivtif3jc +2025-01-06 22:12:54,095 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/ayx5ya39-media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json +2025-01-06 22:12:54,334 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:12:54,408 INFO SenderThread:2121135 [sender.py:send_request_log_artifact():1525] logged artifact run-5jdyg1q2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU0ODAyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDczMDg2OQ==', 'latestArtifact': None}} +2025-01-06 22:12:54,408 DEBUG SenderThread:2121135 [sender.py:send():379] send: files +2025-01-06 22:12:54,408 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json with policy now +2025-01-06 22:12:54,408 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:12:54,409 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:12:54,409 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:54,409 DEBUG SenderThread:2121135 [sender.py:send():379] send: exit +2025-01-06 22:12:54,409 INFO SenderThread:2121135 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:12:54,409 INFO SenderThread:2121135 [sender.py:send_exit():588] handling runtime: 228 +2025-01-06 22:12:54,410 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:12:54,410 INFO SenderThread:2121135 [sender.py:send_exit():594] send defer +2025-01-06 22:12:54,410 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,411 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:12:54,412 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:12:54,412 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:12:54,412 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:12:54,412 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,412 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:12:54,412 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:12:54,412 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:12:54,412 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:12:54,412 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,412 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:12:54,412 INFO HandlerThread:2121135 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:12:54,413 DEBUG SystemMonitor:2121135 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:12:54,413 DEBUG SystemMonitor:2121135 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:12:54,413 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:12:54,413 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:12:54,464 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:12:54,464 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:12:54,464 INFO HandlerThread:2121135 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:12:54,465 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:12:54,465 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:12:54,465 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:12:54,465 DEBUG SenderThread:2121135 [sender.py:send():379] send: stats +2025-01-06 22:12:54,465 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,465 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:12:54,465 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:12:54,465 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:12:54,465 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:12:54,466 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:12:54,466 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:12:54,679 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/ygo9w8k1-media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json +2025-01-06 22:12:54,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:12:54,915 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:12:56,916 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:13:00,582 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:00,582 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:13:00,582 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:13:00,583 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:00,583 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:13:00,583 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:13:00,584 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:00,584 DEBUG SenderThread:2121135 [sender.py:send():379] send: history +2025-01-06 22:13:00,584 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:13:00,584 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:00,584 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:13:00,585 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:13:00,585 DEBUG SenderThread:2121135 [sender.py:send():379] send: summary +2025-01-06 22:13:00,589 INFO SenderThread:2121135 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:13:00,589 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:00,589 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:13:00,589 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:13:00,589 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:00,589 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:13:00,589 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:00,589 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:13:00,593 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:13:00,727 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:13:00,728 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:00,728 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:13:00,728 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:00,728 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:13:00,917 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml +2025-01-06 22:13:00,918 INFO Thread-12 :2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:13:01,391 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:03,149 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:13:03,149 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:03,149 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:03,150 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:13:03,150 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:03,150 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:13:03,150 INFO SenderThread:2121135 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:13:03,152 INFO SenderThread:2121135 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:13:03,154 INFO SenderThread:2121135 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:13:03,165 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:13:03,165 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:03,165 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:13:03,165 DEBUG SenderThread:2121135 [sender.py:send():379] send: artifact +2025-01-06 22:13:03,391 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:03,534 INFO SenderThread:2121135 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 22:13:03,534 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:03,534 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:13:03,534 INFO SenderThread:2121135 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:13:03,918 INFO SenderThread:2121135 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml config.yaml +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json wandb-summary.json +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/requirements.txt requirements.txt +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:13:03,919 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:13:03,921 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log output.log +2025-01-06 22:13:03,921 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/diff.patch diff.patch +2025-01-06 22:13:03,924 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:13:03,924 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 media/videos/eval/3_eval_3_dc101df27963f59b97f2.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 media/videos/eval/4_eval_4_d9c235f657b8c4c13ec2.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 media/videos/eval/1_eval_1_3f6e43c5f90cc9d422ca.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 media/videos/eval/0_eval_0_6426b8e8aadca81f0c9e.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 media/videos/eval/2_eval_2_edc3f417d98fcdf1019e.mp4 +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json media/table/eval/total_success_rate_ci_6_bb805e4eaba65ae7d98f.table.json +2025-01-06 22:13:03,925 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:13:03,926 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:03,928 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:03,928 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:13:03,928 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:03,928 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:13:03,928 INFO SenderThread:2121135 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:13:04,100 INFO wandb-upload_1:2121135 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/config.yaml +2025-01-06 22:13:04,109 INFO wandb-upload_0:2121135 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/wandb-summary.json +2025-01-06 22:13:04,137 INFO wandb-upload_2:2121135 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/requirements.txt +2025-01-06 22:13:04,199 INFO wandb-upload_3:2121135 [upload_job.py:push():130] Uploaded file /tmp/tmpwefveqyswandb/up8yxneq-events.out.tfevents.1736219346.leonmkim-ROG-Strix-G15CS-G15CS.2120949.0 +2025-01-06 22:13:04,201 INFO wandb-upload_4:2121135 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/files/output.log +2025-01-06 22:13:04,392 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:04,392 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:04,401 INFO Thread-11 (_thread_body):2121135 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:13:04,402 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:04,402 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:13:04,402 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:04,402 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:13:04,402 INFO SenderThread:2121135 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:13:04,405 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:13:04,405 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:04,405 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:13:04,405 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:04,405 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:13:04,405 INFO SenderThread:2121135 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:13:04,497 INFO SenderThread:2121135 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:13:04,497 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:13:04,497 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:04,497 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:13:04,497 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:04,497 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:13:04,498 INFO SenderThread:2121135 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:13:04,498 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:13:04,498 INFO HandlerThread:2121135 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:13:04,498 DEBUG SenderThread:2121135 [sender.py:send():379] send: final +2025-01-06 22:13:04,498 DEBUG SenderThread:2121135 [sender.py:send():379] send: footer +2025-01-06 22:13:04,498 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: defer +2025-01-06 22:13:04,498 INFO SenderThread:2121135 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:13:04,499 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:04,499 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:04,499 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:13:04,499 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:13:04,500 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:13:04,503 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:13:04,503 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:13:04,504 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:13:04,556 DEBUG SenderThread:2121135 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:13:04,557 INFO MainThread:2121135 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:13:04,557 INFO MainThread:2121135 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:13:04,557 INFO MainThread:2121135 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:13:04,557 DEBUG HandlerThread:2121135 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:13:04,557 INFO HandlerThread:2121135 [handler.py:finish():882] shutting down handler +2025-01-06 22:13:05,503 INFO WriterThread:2121135 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/run-5jdyg1q2.wandb +2025-01-06 22:13:05,556 INFO SenderThread:2121135 [sender.py:finish():1615] shutting down sender +2025-01-06 22:13:05,557 INFO SenderThread:2121135 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:13:05,557 INFO SenderThread:2121135 [file_pusher.py:join():175] waiting for file pusher diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug.log b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b740ba16729905aa26e63a2a899084720d410f5d --- /dev/null +++ b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Configure stats pid to 2120949 +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/settings +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:09:05,575 INFO MainThread:2120949 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug.log +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/logs/debug-internal.log +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2'} +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:init():619] starting backend +2025-01-06 22:09:05,576 INFO MainThread:2120949 [wandb_init.py:init():623] setting up manager +2025-01-06 22:09:05,580 INFO MainThread:2120949 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:09:05,581 INFO MainThread:2120949 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:09:05,596 INFO MainThread:2120949 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:09:05,602 INFO MainThread:2120949 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:09:05,857 INFO MainThread:2120949 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:09:05,899 INFO MainThread:2120949 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:09:05,899 INFO MainThread:2120949 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:09:06,244 INFO MainThread:2120949 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:09:06,244 INFO MainThread:2120949 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:09:06,244 INFO MainThread:2120949 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:09:06,245 INFO MainThread:2120949 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:09:06,246 INFO MainThread:2120949 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:09:06,246 INFO MainThread:2120949 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/5jdyg1q2/tb, True +2025-01-06 22:09:21,433 INFO MainThread:2120949 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:13:05,558 WARNING MsgRouterThr:2120949 [router.py:message_loop():77] message_loop has been closed diff --git a/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/run-5jdyg1q2.wandb b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/run-5jdyg1q2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0e778b560a8b7cc520d7f2b05060488926c66c2d Binary files /dev/null and b/5jdyg1q2/wandb/run-20250106_220905-5jdyg1q2/run-5jdyg1q2.wandb differ diff --git a/5qmtbx9d/episode_rosbags/episode_1_2025-01-07-20-46-02.bag b/5qmtbx9d/episode_rosbags/episode_1_2025-01-07-20-46-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..4133d9fd4147d09904b5d5a2991b0da878c73ee2 --- /dev/null +++ b/5qmtbx9d/episode_rosbags/episode_1_2025-01-07-20-46-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33d603e2a56aa0b1c4839f4d408b8fd72d9a3f0b891b06738c24677c7b149259 +size 1441433412 diff --git a/5w3hakcj/episode_rosbags/cam_tf_world.npy b/5w3hakcj/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/5w3hakcj/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/5w3hakcj/episode_rosbags/depth_K.npy b/5w3hakcj/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/5w3hakcj/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/5w3hakcj/eval_robot.log b/5w3hakcj/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..5e4de3bc95179c22a589f7617c306022b0a6d297 --- /dev/null +++ b/5w3hakcj/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 17:18:43,836][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:18:43,838][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/9fviy2t8/episode_rosbags/episode_3_2025-01-06-17-26-27.bag b/9fviy2t8/episode_rosbags/episode_3_2025-01-06-17-26-27.bag new file mode 100644 index 0000000000000000000000000000000000000000..0461623365330b19efcadd3c20c7217f8ccae652 --- /dev/null +++ b/9fviy2t8/episode_rosbags/episode_3_2025-01-06-17-26-27.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d280724ea0dba13cc9c679a5371979e8326f586dc3688c385028b87547eaee3b +size 1406720570 diff --git a/9fviy2t8/episode_rosbags/episode_4_2025-01-06-17-27-09.bag b/9fviy2t8/episode_rosbags/episode_4_2025-01-06-17-27-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..69877c61cb63540fff010834ad776127b1d28b3e --- /dev/null +++ b/9fviy2t8/episode_rosbags/episode_4_2025-01-06-17-27-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f5a48ad3bd1948434dab9456a28fd3de94222fc3971b2fd5cedc8cf09645d02 +size 1409712623 diff --git a/9jw2ghih/episode_rosbags/episode_4_2025-01-06-20-43-59.bag b/9jw2ghih/episode_rosbags/episode_4_2025-01-06-20-43-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..792db96f0aa07bc416903bc9bac0aaadd55ce64f --- /dev/null +++ b/9jw2ghih/episode_rosbags/episode_4_2025-01-06-20-43-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f84a1f21013712ee6127c38d6908d94ea5a347f95dd194be71308c61746085c +size 1412703580 diff --git a/nyyc47dd/episode_rosbags/episode_1_2025-01-07-17-10-47.bag b/nyyc47dd/episode_rosbags/episode_1_2025-01-07-17-10-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..e3de8fef9f84a73ac89ac46293fc3a06a98dd0fd --- /dev/null +++ b/nyyc47dd/episode_rosbags/episode_1_2025-01-07-17-10-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:257c1abb7efb44b03893e9348359851fa58595c98a1be68b9e524e35b0d11549 +size 1410413138 diff --git a/o9qcewu3/.hydra/config.yaml b/o9qcewu3/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/o9qcewu3/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/o9qcewu3/.hydra/hydra.yaml b/o9qcewu3/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51d2b571d47015a632e2405328f8a1cf1cc15425 --- /dev/null +++ b/o9qcewu3/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/o9qcewu3/.hydra/overrides.yaml b/o9qcewu3/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/o9qcewu3/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/o9qcewu3/episode_rosbags/aligned_depth_to_color_K.npy b/o9qcewu3/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/o9qcewu3/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/o9qcewu3/episode_rosbags/color_K.npy b/o9qcewu3/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/o9qcewu3/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/o9qcewu3/episode_rosbags/depth_K.npy b/o9qcewu3/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/o9qcewu3/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/o9qcewu3/episode_rosbags/episode_0_2025-01-07-20-50-16.bag b/o9qcewu3/episode_rosbags/episode_0_2025-01-07-20-50-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..26849595a98b256816a3d7f9739886cd67f18da8 --- /dev/null +++ b/o9qcewu3/episode_rosbags/episode_0_2025-01-07-20-50-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c349869a8e1c6948da417d7dae7073c89b61f1a91a0fb3bf9203f03aeda5ef +size 1420235500 diff --git a/o9qcewu3/episode_rosbags/episode_1_2025-01-07-20-50-59.bag b/o9qcewu3/episode_rosbags/episode_1_2025-01-07-20-50-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..99783d3d78a8245a0513185aae29449c55c68f63 --- /dev/null +++ b/o9qcewu3/episode_rosbags/episode_1_2025-01-07-20-50-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:092ebd3e1d75e3c791a9d689bf15a9f4ce4fc5deee8a9bf5b6bc648d8c6344bb +size 1412178555 diff --git a/o9qcewu3/episode_rosbags/episode_2_2025-01-07-20-51-45.bag b/o9qcewu3/episode_rosbags/episode_2_2025-01-07-20-51-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..0fb1913b926b4fc3b68fa40a5626e67ab95e8133 --- /dev/null +++ b/o9qcewu3/episode_rosbags/episode_2_2025-01-07-20-51-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8051f8a4680f331598b6d54ae99ceb0f1ed7b7a067b97485197ba8c953fccd2 +size 1415495370 diff --git a/o9qcewu3/episode_rosbags/episode_3_2025-01-07-20-52-32.bag b/o9qcewu3/episode_rosbags/episode_3_2025-01-07-20-52-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..260d223cff93c47847144ffd03b88cbda547372b --- /dev/null +++ b/o9qcewu3/episode_rosbags/episode_3_2025-01-07-20-52-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66b9690ed7b61a0950a48a0cf6dbc2fc97b9da7e29c11c19b58ac6a432350c7c +size 1407935868 diff --git a/o9qcewu3/eval_video/0_eval.mp4 b/o9qcewu3/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d2c2c225ec6e083c88bdcd4424e248d90e256b4 --- /dev/null +++ b/o9qcewu3/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0760969f5acfd52720a3e3072d3bdbd486f3f57bee2fe5de8203e5300f83b49a +size 1136099 diff --git a/o9qcewu3/eval_video/1_eval.mp4 b/o9qcewu3/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5521e07da2ace0325f5d5cfb13bf8a5914f622f8 --- /dev/null +++ b/o9qcewu3/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6a2c2926ca8c08bde56e2bee7c22f47cfca2a06833064b0e7d21793e4bd85e +size 1143216 diff --git a/o9qcewu3/eval_video/2_eval.mp4 b/o9qcewu3/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..785feb1b99fc40c369ee2931928e83ea8a6ed912 --- /dev/null +++ b/o9qcewu3/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ed1386a144a7cf88bb2e9788e0605909d7c93931a950927efa799acd16ddbc7 +size 1131224 diff --git a/o9qcewu3/eval_video/3_eval.mp4 b/o9qcewu3/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2a7e8f8ef831a7bddf488da02e34b1283fe11b80 --- /dev/null +++ b/o9qcewu3/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbf2275ce0ac22627299044cd7560c9382c060d0d20bbac7a9f9711f54e335e8 +size 1151672 diff --git a/o9qcewu3/eval_video/4_eval.mp4 b/o9qcewu3/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07c2eff2a2a0831af802b8df46ec8f2597143709 --- /dev/null +++ b/o9qcewu3/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1992567de3417dc1496f4d125be64105835ae9f18a5e8e053a7a2afbd6ddce74 +size 1132300 diff --git a/o9qcewu3/tb/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 b/o9qcewu3/tb/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 new file mode 100644 index 0000000000000000000000000000000000000000..4c856766d4596f3bc3dce7c955c131e9059a6644 --- /dev/null +++ b/o9qcewu3/tb/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c87c249f38db3080672a29a9dc67c07172f12e11647f09f64c7526934562835a +size 6539 diff --git a/o9qcewu3/wandb/debug-internal.log b/o9qcewu3/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a8bd1f9084e2aea0690041408791739857ebac4f --- /dev/null +++ b/o9qcewu3/wandb/debug-internal.log @@ -0,0 +1,760 @@ +2025-01-07 20:49:54,251 INFO StreamThr :2231710 [internal.py:wandb_internal():85] W&B internal server running at pid: 2231710, started at: 2025-01-07 20:49:54.251317 +2025-01-07 20:49:54,252 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:49:54,266 INFO WriterThread:2231710 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/run-o9qcewu3.wandb +2025-01-07 20:49:54,266 DEBUG SenderThread:2231710 [sender.py:send():379] send: header +2025-01-07 20:49:54,274 DEBUG SenderThread:2231710 [sender.py:send():379] send: run +2025-01-07 20:49:54,555 INFO SenderThread:2231710 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files +2025-01-07 20:49:54,555 INFO SenderThread:2231710 [sender.py:_start_run_threads():1188] run started: o9qcewu3 with start time 1736300994.25195 +2025-01-07 20:49:54,570 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:49:54,570 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:49:54,646 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:49:54,651 DEBUG HandlerThread:2231710 [system_info.py:__init__():26] System info init +2025-01-07 20:49:54,651 DEBUG HandlerThread:2231710 [system_info.py:__init__():41] System info init done +2025-01-07 20:49:54,651 INFO HandlerThread:2231710 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:49:54,651 INFO SystemMonitor:2231710 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:49:54,652 INFO HandlerThread:2231710 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:49:54,652 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:49:54,653 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:49:54,653 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:49:54,654 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:49:54,655 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:49:54,664 DEBUG HandlerThread:2231710 [system_info.py:probe():152] Probing system +2025-01-07 20:49:54,666 DEBUG HandlerThread:2231710 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:49:54,671 DEBUG HandlerThread:2231710 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:49:54,671 DEBUG HandlerThread:2231710 [system_info.py:probe():200] Probing system done +2025-01-07 20:49:54,671 DEBUG HandlerThread:2231710 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:49:54.665027', 'startedAt': '2025-01-08T01:49:54.237296', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.7419999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.923, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 613.4297180175781}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:49:54,671 INFO HandlerThread:2231710 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:49:54,671 INFO HandlerThread:2231710 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:49:54,671 DEBUG HandlerThread:2231710 [system_info.py:_save_code():44] Saving code +2025-01-07 20:49:54,672 DEBUG HandlerThread:2231710 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:49:54,673 DEBUG HandlerThread:2231710 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:49:54,740 DEBUG HandlerThread:2231710 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:49:54,740 INFO HandlerThread:2231710 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:49:54,743 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:49:54,743 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:49:54,743 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:49:54,744 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:49:54,996 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/91k8dpks-wandb-metadata.json +2025-01-07 20:49:55,013 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:49:55,013 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:49:55,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:49:55,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:49:55,033 INFO wandb-upload_2:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/bhe3gypl-diff.patch +2025-01-07 20:49:55,036 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/3z9wb4zl-code/FISH/eval_robot.py +2025-01-07 20:49:55,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:55,057 INFO HandlerThread:2231710 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/tb" + save: true +} +control { + relay_id: "b5f985b5bb4a435d9f6388b1eebaee97" +} +_info { + stream_id: "o9qcewu3" +} + +2025-01-07 20:49:55,057 DEBUG HandlerThread:2231710 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:49:55,092 DEBUG SenderThread:2231710 [sender.py:send():379] send: telemetry +2025-01-07 20:49:55,108 DEBUG SenderThread:2231710 [sender.py:send():379] send: tbrecord +2025-01-07 20:49:55,109 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:49:55,109 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 with policy live +2025-01-07 20:49:55,283 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/g7dvd66t-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code/FISH/eval_robot.py +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/requirements.txt +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/diff.patch +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-metadata.json +2025-01-07 20:49:55,558 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code +2025-01-07 20:49:55,558 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code/FISH +2025-01-07 20:49:56,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:57,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:57,556 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:49:58,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:59,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:00,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:00,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:00,512 DEBUG SenderThread:2231710 [sender.py:send():379] send: config +2025-01-07 20:50:01,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:01,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:02,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:03,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:03,558 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:04,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:05,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:05,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:05,558 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:06,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:07,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:07,559 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:08,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:09,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:09,560 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:10,013 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:50:10,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:50:10,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:10,168 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:11,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:12,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:13,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:14,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:15,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:15,561 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:16,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:16,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:17,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:18,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:19,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:19,562 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:20,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:21,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:21,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:22,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:23,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:24,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:25,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:50:25,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:50:25,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:26,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:26,157 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:26,564 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml +2025-01-07 20:50:27,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:28,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:29,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:30,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:31,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:32,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:32,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:33,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:34,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:35,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:36,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:37,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:37,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:38,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:39,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:40,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:50:40,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:50:40,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:41,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:42,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:42,123 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:43,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:44,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:45,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:45,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:50:45,118 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:50:45,118 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 with policy now +2025-01-07 20:50:45,118 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:50:45,119 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:50:45,120 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:50:45,337 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/x2ibtcl5-media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 +2025-01-07 20:50:45,569 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:45,569 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:50:45,569 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 +2025-01-07 20:50:45,569 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:50:45,570 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos +2025-01-07 20:50:45,570 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media +2025-01-07 20:50:45,849 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/lizyi1pp-media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 +2025-01-07 20:50:46,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:47,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:47,570 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:48,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:48,120 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:49,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:50,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:51,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:51,571 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:52,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:53,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:53,572 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:53,610 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:54,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:54,655 DEBUG SystemMonitor:2231710 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:50:54,656 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:50:55,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:50:55,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:50:55,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:56,024 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:56,573 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:50:56,818 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/k72x4l9r-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:50:57,024 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:57,573 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:58,024 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:59,024 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:59,193 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:59,574 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:00,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:01,039 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:01,575 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:02,039 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:03,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:04,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:05,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:05,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:06,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:07,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:08,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:09,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:10,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:51:10,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:51:10,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:10,127 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:11,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:12,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:13,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:14,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:15,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:16,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:16,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:17,042 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:18,042 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:19,042 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:20,042 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:21,043 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:22,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:22,045 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:23,045 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:24,045 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:24,656 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:51:25,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:51:25,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:51:25,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:26,045 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:27,046 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:27,614 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:27,851 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:51:27,851 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 with policy now +2025-01-07 20:51:27,855 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:51:27,860 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:51:27,860 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:51:27,860 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:51:28,047 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:28,098 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/99n9bpv7-media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 +2025-01-07 20:51:28,589 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:51:28,589 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 +2025-01-07 20:51:28,590 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:51:28,864 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/ex6vn3j5-media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 +2025-01-07 20:51:29,050 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:29,589 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:30,050 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:31,050 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:32,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:32,645 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:33,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:33,591 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:34,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:35,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:35,592 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:36,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:37,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:38,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:38,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:39,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:40,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:51:40,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:51:40,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:41,050 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:42,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:42,462 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:51:42,462 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:51:42,462 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:51:42,594 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:51:42,594 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:51:43,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:43,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:43,594 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:44,052 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:45,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:45,595 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:46,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:47,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:47,596 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:48,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:48,919 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:49,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:50,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:51,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:52,053 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:53,053 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:54,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:54,053 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:54,656 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:51:55,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:51:55,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:51:55,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:56,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:57,053 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:58,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:59,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:59,157 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:00,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:01,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:02,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:03,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:04,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:05,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:05,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:06,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:07,055 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:08,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:09,055 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:10,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:52:10,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:52:10,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:10,090 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:11,055 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:12,055 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:13,059 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:13,604 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:14,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:14,197 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:52:14,197 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 with policy now +2025-01-07 20:52:14,200 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:52:14,201 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:52:14,201 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:52:14,201 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:52:14,416 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/azqrmhf1-media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 +2025-01-07 20:52:14,604 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:52:14,605 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 +2025-01-07 20:52:14,605 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:52:14,862 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/u2ogjcly-media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 +2025-01-07 20:52:15,061 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:15,605 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:16,028 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:16,063 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:16,605 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:17,064 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:18,077 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:19,066 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:19,606 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:20,069 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:21,064 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:21,885 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:22,065 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:23,066 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:23,607 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:24,065 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:24,657 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:52:25,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:52:25,020 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:52:25,067 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:26,072 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:27,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:27,172 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:28,072 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:28,609 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:52:28,850 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/xrx1rliz-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:52:29,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:29,298 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:52:29,298 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:52:29,299 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:52:29,609 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:52:29,610 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:52:29,610 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:30,081 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:31,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:31,610 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:32,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:32,428 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:33,073 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:33,611 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:34,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:35,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:35,612 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:36,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:37,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:38,026 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:38,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:39,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:40,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:52:40,018 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:52:40,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:41,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:42,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:43,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:43,138 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:44,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:45,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:46,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:47,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:48,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:48,685 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:49,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:49,616 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:50,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:51,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:51,617 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:52,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:53,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:53,618 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:53,897 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:54,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:54,657 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:52:55,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:52:55,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:52:55,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:56,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:57,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:58,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:59,077 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:59,170 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:00,080 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:01,097 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:01,183 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:53:01,183 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 with policy now +2025-01-07 20:53:01,190 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:53:01,191 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:53:01,191 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:53:01,192 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:53:01,397 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/ptvy46ax-media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 +2025-01-07 20:53:01,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:53:01,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:01,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 +2025-01-07 20:53:01,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:53:01,878 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/shg99opq-media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 +2025-01-07 20:53:02,099 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:02,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:03,107 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:03,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:04,103 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:04,205 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:05,105 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:06,108 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:07,110 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:07,623 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:08,106 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:09,107 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:09,623 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:09,845 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:10,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:53:10,017 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:53:10,108 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:11,111 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:12,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:13,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:14,109 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:15,033 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:15,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:16,112 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:17,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:17,367 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:53:17,367 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:53:17,368 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:53:17,626 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:53:17,626 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:53:18,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:19,120 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:19,627 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:20,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:20,847 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:21,112 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:22,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:23,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:23,628 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:24,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:24,658 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:53:25,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:53:25,016 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:53:25,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:26,100 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:26,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:27,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:28,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:29,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:30,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:31,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:32,031 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:32,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:33,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:34,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:35,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:36,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:37,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:38,031 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:38,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:39,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:40,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:53:40,017 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:53:40,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:41,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:42,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:43,080 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:43,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:44,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:45,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:46,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:47,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:48,094 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:48,118 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:49,117 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:53:49,117 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 with policy now +2025-01-07 20:53:49,118 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:53:49,118 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:53:49,118 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:53:49,121 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:53:49,125 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:49,327 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/rkjgfxwo-media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 +2025-01-07 20:53:49,640 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:53:49,641 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:49,641 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 +2025-01-07 20:53:49,641 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:53:49,946 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/afmz54iy-media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 +2025-01-07 20:53:50,120 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:51,126 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:51,641 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:52,122 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:53,124 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:54,009 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:54,133 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:54,658 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:53:55,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:53:55,022 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:53:55,126 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:55,643 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:56,187 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:57,195 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:57,643 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:58,191 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:59,195 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:59,853 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:00,192 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:01,195 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:02,199 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:03,207 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:04,197 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:04,428 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:54:04,428 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:04,429 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:04,595 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:54:04,595 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval +2025-01-07 20:54:04,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table +2025-01-07 20:54:04,723 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:54:04,826 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/4epar6xy-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:54:04,920 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/b057mig5-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:05,115 INFO wandb-upload_2:2231710 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvskn9n77 +2025-01-07 20:54:05,163 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:54:05,171 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:54:05,197 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:05,533 INFO SenderThread:2231710 [sender.py:send_request_log_artifact():1525] logged artifact run-o9qcewu3-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQyNTk3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0Mjg0MQ==', 'latestArtifact': None}} +2025-01-07 20:54:05,533 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:54:05,533 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 20:54:05,533 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:05,534 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:54:05,534 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,534 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,535 DEBUG SenderThread:2231710 [sender.py:send():379] send: telemetry +2025-01-07 20:54:05,535 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,539 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,539 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,539 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,539 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,540 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,540 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,540 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,540 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,540 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,541 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,541 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,541 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,541 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,542 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,542 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,542 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,542 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,543 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,543 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,543 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,544 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,544 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:54:05,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:05,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:05,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:05,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json +2025-01-07 20:54:05,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval +2025-01-07 20:54:05,781 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/p2qyvtdm-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:54:05,823 INFO wandb-upload_2:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/6fmddz6w-media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json +2025-01-07 20:54:05,930 INFO wandb-upload_1:2231710 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzwgvpbvm +2025-01-07 20:54:06,197 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:06,343 INFO SenderThread:2231710 [sender.py:send_request_log_artifact():1525] logged artifact run-o9qcewu3-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQyNTk5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0Mjg0Ng==', 'latestArtifact': None}} +2025-01-07 20:54:06,344 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:54:06,344 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json with policy now +2025-01-07 20:54:06,344 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:54:06,344 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:06,345 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:06,588 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/ga9zu2ca-media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json +2025-01-07 20:54:06,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:07,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:07,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:08,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:09,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:10,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:54:10,018 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:54:10,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:11,173 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:11,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:12,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:13,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:13,665 DEBUG SenderThread:2231710 [sender.py:send():379] send: exit +2025-01-07 20:54:13,665 INFO SenderThread:2231710 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:send_exit():588] handling runtime: 259 +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:send_exit():594] send defer +2025-01-07 20:54:13,666 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,666 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:54:13,666 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:54:13,667 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,667 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:54:13,667 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:13,667 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:54:13,667 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:54:13,667 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,667 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:54:13,667 INFO HandlerThread:2231710 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:54:13,667 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:54:13,667 DEBUG SystemMonitor:2231710 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:54:13,668 DEBUG SystemMonitor:2231710 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:54:13,668 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:54:13,720 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:54:13,720 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:54:13,720 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:54:13,721 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:13,721 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:54:13,721 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:54:13,721 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:54:13,721 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,721 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:54:13,721 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:13,722 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:54:13,722 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:54:13,722 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,722 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:54:14,650 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:15,650 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:17,651 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml +2025-01-07 20:54:20,435 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:20,435 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:20,435 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:54:20,437 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:20,437 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:54:20,437 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:54:20,437 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:20,437 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:54:20,437 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:20,437 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:20,437 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:54:20,438 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:20,438 DEBUG SenderThread:2231710 [sender.py:send():379] send: summary +2025-01-07 20:54:20,441 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:20,441 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:20,441 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:54:20,441 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:54:20,442 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:20,442 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:54:20,442 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:20,442 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:54:20,442 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:54:20,442 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:20,442 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:20,442 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:54:20,442 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:20,443 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:54:20,652 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:20,667 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:22,095 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:54:22,095 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:22,095 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:22,096 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:54:22,096 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:22,096 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:54:22,096 INFO SenderThread:2231710 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:54:22,099 INFO SenderThread:2231710 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:54:22,101 INFO SenderThread:2231710 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:54:22,114 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:54:22,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:22,115 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:54:22,115 DEBUG SenderThread:2231710 [sender.py:send():379] send: artifact +2025-01-07 20:54:22,311 INFO SenderThread:2231710 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:54:22,311 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:22,311 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:54:22,311 INFO SenderThread:2231710 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:54:22,653 INFO SenderThread:2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:22,653 INFO SenderThread:2231710 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml config.yaml +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json wandb-summary.json +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/requirements.txt requirements.txt +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log output.log +2025-01-07 20:54:22,659 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/diff.patch diff.patch +2025-01-07 20:54:22,659 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:22,660 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:54:22,661 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 +2025-01-07 20:54:22,661 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 +2025-01-07 20:54:22,661 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 +2025-01-07 20:54:22,661 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 +2025-01-07 20:54:22,662 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 +2025-01-07 20:54:22,662 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json +2025-01-07 20:54:22,662 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:54:22,663 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:54:22,663 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:22,663 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:54:22,663 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:22,663 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:54:22,663 INFO SenderThread:2231710 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:54:22,668 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:22,668 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:22,843 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/requirements.txt +2025-01-07 20:54:22,847 INFO wandb-upload_2:2231710 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:22,852 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml +2025-01-07 20:54:22,879 INFO wandb-upload_3:2231710 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:22,938 INFO wandb-upload_4:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/k6ad1g9x-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:23,139 INFO Thread-11 (_thread_body):2231710 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:54:23,139 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:23,139 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:54:23,139 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:23,139 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:54:23,139 INFO SenderThread:2231710 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:54:23,140 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:54:23,141 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:23,141 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:54:23,141 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:23,141 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:54:23,141 INFO SenderThread:2231710 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:54:23,364 INFO SenderThread:2231710 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:54:23,365 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:54:23,365 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:23,365 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:54:23,365 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:23,365 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:54:23,365 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:54:23,365 DEBUG SenderThread:2231710 [sender.py:send():379] send: final +2025-01-07 20:54:23,365 DEBUG SenderThread:2231710 [sender.py:send():379] send: footer +2025-01-07 20:54:23,365 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:23,365 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:54:23,366 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:23,366 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:54:23,366 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:23,366 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:54:23,367 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:23,367 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:54:23,367 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:23,367 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:54:23,368 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:54:23,368 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:23,436 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:23,436 INFO MainThread:2231710 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:54:23,437 INFO MainThread:2231710 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:54:23,437 INFO MainThread:2231710 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:54:23,437 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:54:23,437 INFO HandlerThread:2231710 [handler.py:finish():882] shutting down handler +2025-01-07 20:54:24,367 INFO WriterThread:2231710 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/run-o9qcewu3.wandb +2025-01-07 20:54:24,436 INFO SenderThread:2231710 [sender.py:finish():1615] shutting down sender +2025-01-07 20:54:24,436 INFO SenderThread:2231710 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:54:24,436 INFO SenderThread:2231710 [file_pusher.py:join():175] waiting for file pusher diff --git a/o9qcewu3/wandb/debug.log b/o9qcewu3/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..97f81e56e3ba0ca3d3cb38f32ffbe7c18f0a03e0 --- /dev/null +++ b/o9qcewu3/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Configure stats pid to 2231543 +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/settings +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug.log +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug-internal.log +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:49:54,244 INFO MainThread:2231543 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3'} +2025-01-07 20:49:54,244 INFO MainThread:2231543 [wandb_init.py:init():619] starting backend +2025-01-07 20:49:54,244 INFO MainThread:2231543 [wandb_init.py:init():623] setting up manager +2025-01-07 20:49:54,250 INFO MainThread:2231543 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:49:54,251 INFO MainThread:2231543 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:49:54,266 INFO MainThread:2231543 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:49:54,272 INFO MainThread:2231543 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:49:54,569 INFO MainThread:2231543 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:49:54,641 INFO MainThread:2231543 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:49:54,641 INFO MainThread:2231543 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:49:55,015 INFO MainThread:2231543 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:49:55,015 INFO MainThread:2231543 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:49:55,015 INFO MainThread:2231543 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:49:55,016 INFO MainThread:2231543 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:49:55,018 INFO MainThread:2231543 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:49:55,018 INFO MainThread:2231543 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/tb, True +2025-01-07 20:50:00,511 INFO MainThread:2231543 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:54:24,438 WARNING MsgRouterThr:2231543 [router.py:message_loop():77] message_loop has been closed diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code/FISH/eval_robot.py b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fcb98e3e6a82d49ceb36312a6ff7ae10a12e09a --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1009_1' +wandb_notes: + desc: null + value: 1009_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736300994 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/diff.patch b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c0161c26d9b20c7c85242592d6833c2df9767742 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/diff.patch @@ -0,0 +1,20 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..aa5b9b9 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,13 +134,13 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' +-# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e39d956ac399d2b301a4496968f2021ec86ac080 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[60, 60, 1.0, 0.9982039629416646, 1, 0.9455429463687541, 1, 0.9530260766881642, 1, 1, 1]]} \ No newline at end of file diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d2c2c225ec6e083c88bdcd4424e248d90e256b4 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0760969f5acfd52720a3e3072d3bdbd486f3f57bee2fe5de8203e5300f83b49a +size 1136099 diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5521e07da2ace0325f5d5cfb13bf8a5914f622f8 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6a2c2926ca8c08bde56e2bee7c22f47cfca2a06833064b0e7d21793e4bd85e +size 1143216 diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..785feb1b99fc40c369ee2931928e83ea8a6ed912 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ed1386a144a7cf88bb2e9788e0605909d7c93931a950927efa799acd16ddbc7 +size 1131224 diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2a7e8f8ef831a7bddf488da02e34b1283fe11b80 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbf2275ce0ac22627299044cd7560c9382c060d0d20bbac7a9f9711f54e335e8 +size 1151672 diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07c2eff2a2a0831af802b8df46ec8f2597143709 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1992567de3417dc1496f4d125be64105835ae9f18a5e8e053a7a2afbd6ddce74 +size 1132300 diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9b6d08615a464feff576f793f40397aad82dd869 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log @@ -0,0 +1,280 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736301000.512644]: resetting environment +[INFO] [1736301000.516983]: cleared current plan +[INFO] [1736301000.519045]: moving to home +[INFO] [1736301001.522511]: reached home +[INFO] [1736301001.523790]: reset action history +[INFO] [1736301004.230469]: environment reset +Starting episode 0 +[INFO] [1736301004.232432]: resetting environment +[INFO] [1736301004.234940]: cleared current plan +[INFO] [1736301004.235752]: moving to home +[INFO] [1736301005.237264]: reached home +[INFO] [1736301005.241528]: reset action history +[INFO] [1736301008.000616]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736301013.199929]: resetting environment +[INFO] [1736301013.207364]: cleared current plan +[INFO] [1736301013.208958]: moving to home +[INFO] [1736301014.212310]: reached home +[INFO] [1736301014.214204]: reset action history +[INFO] [1736301016.922392]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/episode_rosbags/episode_0_2025-01-07-20-50-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2231543.log +started roslaunch server http://158.130.50.37:39175/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2231811] +process[print_text-2]: started with pid [2231835] +process[rosbag_record-3]: started with pid [2231836] +started bagging! +[INFO] [1736301044.097514]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736301044.880724]: Stored episode 1. +[INFO] [1736301044.882535]: Saving video... +[INFO] [1736301045.074030]: Video saved! +Starting episode 1 +[INFO] [1736301045.096380]: resetting environment +[INFO] [1736301045.100289]: cleared current plan +[INFO] [1736301045.100493]: moving to home +[INFO] [1736301048.805199]: reached home +[INFO] [1736301048.806189]: reset action history +[INFO] [1736301051.564459]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736301056.147511]: resetting environment +[INFO] [1736301056.149929]: cleared current plan +[INFO] [1736301056.150297]: moving to home +[INFO] [1736301057.151717]: reached home +[INFO] [1736301057.152543]: reset action history +[INFO] [1736301059.857983]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/episode_rosbags/episode_1_2025-01-07-20-50-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2231543.log +started roslaunch server http://158.130.50.37:35393/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2231946] +process[print_text-5]: started with pid [2231947] +process[rosbag_record-6]: started with pid [2231971] +started bagging! +[INFO] [1736301086.816168]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736301087.614039]: Stored episode 2. +[INFO] [1736301087.614990]: Saving video... +[INFO] [1736301087.815443]: Video saved! +Starting episode 2 +[INFO] [1736301087.856964]: resetting environment +[INFO] [1736301087.883138]: cleared current plan +[INFO] [1736301087.884480]: moving to home +[INFO] [1736301091.597111]: reached home +[INFO] [1736301091.597923]: reset action history +[INFO] [1736301094.375889]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736301101.964355]: resetting environment +[INFO] [1736301101.973203]: cleared current plan +[INFO] [1736301101.973534]: moving to home +[INFO] [1736301102.975444]: reached home +[INFO] [1736301102.975884]: reset action history +[INFO] [1736301105.681517]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/episode_rosbags/episode_2_2025-01-07-20-51-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2231543.log +started roslaunch server http://158.130.50.37:38809/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2232082] +process[print_text-8]: started with pid [2232083] +process[rosbag_record-9]: started with pid [2232084] +started bagging! +[INFO] [1736301132.626678]: Storing episode... +[pub_text-7] killing on exit +[print_text-8] killing on exit +[rosbag_record-9] killing on exit +[INFO] [1736301133.475084]: Stored episode 3. +[INFO] [1736301133.476750]: Saving video... +[INFO] [1736301134.158431]: Video saved! +Starting episode 3 +[INFO] [1736301134.205600]: resetting environment +[INFO] [1736301134.224622]: cleared current plan +[INFO] [1736301134.224881]: moving to home +[WARN] [1736301133.964184]: Plan exhausted +[WARN] [1736301134.002649]: Plan exhausted +[WARN] [1736301134.042527]: Plan exhausted +[WARN] [1736301134.081615]: Plan exhausted +[WARN] [1736301134.134357]: Plan exhausted +[WARN] [1736301134.164003]: Plan exhausted +[WARN] [1736301134.203706]: Plan exhausted +[INFO] [1736301138.036226]: reached home +[INFO] [1736301138.044836]: reset action history +[INFO] [1736301140.838388]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736301148.631803]: resetting environment +[INFO] [1736301148.645259]: cleared current plan +[INFO] [1736301148.647944]: moving to home +[INFO] [1736301149.654567]: reached home +[INFO] [1736301149.658885]: reset action history +[INFO] [1736301152.428341]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/episode_rosbags/episode_3_2025-01-07-20-52-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2231543.log +started roslaunch server http://158.130.50.37:42553/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2232241] +process[print_text-11]: started with pid [2232242] +process[rosbag_record-12]: started with pid [2232243] +started bagging! +For topic gripper_width: timestamp difference is 109720050 for nearest: 1736301167869321643 - target: 1736301167759601593 at idx 197 +For topic gripper_width: timestamp difference is 110403217 for nearest: 1736301168669319786 - target: 1736301168558916569 at idx 207 +For topic gripper_width: timestamp difference is 109076523 for nearest: 1736301169469356798 - target: 1736301169360280275 at idx 217 +For topic gripper_width: timestamp difference is -102122691 for nearest: 1736301170902655456 - target: 1736301171004778147 at idx 235 +For topic gripper_width: timestamp difference is -100243862 for nearest: 1736301171702656214 - target: 1736301171802900076 at idx 243 +[INFO] [1736301179.632622]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736301180.414617]: Stored episode 4. +[INFO] [1736301180.420377]: Saving video... +[WARN] [1736301180.963593]: Plan exhausted +[WARN] [1736301181.001612]: Plan exhausted +[WARN] [1736301181.041783]: Plan exhausted +[WARN] [1736301181.094239]: Plan exhausted +[WARN] [1736301181.121963]: Plan exhausted +[WARN] [1736301181.161605]: Plan exhausted +[WARN] [1736301181.201608]: Plan exhausted +[INFO] [1736301181.145277]: Video saved! +Starting episode 4 +[INFO] [1736301181.191929]: resetting environment +[INFO] [1736301181.204106]: cleared current plan +[INFO] [1736301181.204331]: moving to home +[INFO] [1736301185.006722]: reached home +[INFO] [1736301185.012436]: reset action history +[INFO] [1736301187.801651]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736301197.031518]: resetting environment +[INFO] [1736301197.048859]: cleared current plan +[INFO] [1736301197.052825]: moving to home +[INFO] [1736301198.061301]: reached home +[INFO] [1736301198.063175]: reset action history +[INFO] [1736301200.845823]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/episode_rosbags/episode_4_2025-01-07-20-53-20.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2231543.log +started roslaunch server http://158.130.50.37:34879/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2232384] +process[print_text-14]: started with pid [2232385] +process[rosbag_record-15]: started with pid [2232386] +started bagging! +[INFO] [1736301228.043435]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit[print_text-14] killing on exit +[INFO] [1736301228.819184]: Stored episode 5. +[INFO] [1736301228.827162]: Saving video... +[INFO] [1736301229.088464]: Video saved! +[INFO] [1736301229.119617]: resetting environment +[INFO] [1736301229.150864]: cleared current plan +[INFO] [1736301229.153233]: moving to home +[INFO] [1736301232.962179]: reached home +[INFO] [1736301232.966541]: reset action history +[INFO] [1736301235.810959]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/requirements.txt b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-metadata.json b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d1c1d8ed772ff7b42446f6248df04d6d4e64eeb9 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:49:54.665027", + "startedAt": "2025-01-08T01:49:54.237296", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.7419999999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.923, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 613.4297180175781 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..650a97facca4ba538a03b4d6eb52b24c97c51efe --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "0760969f5acfd52720a3e3072d3bdbd486f3f57bee2fe5de8203e5300f83b49a", "size": 1136099, "path": "media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4"}, "global_step": 665, "_timestamp": 1736301245.0661805, "_runtime": 250.81423044204712, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "ee6a2c2926ca8c08bde56e2bee7c22f47cfca2a06833064b0e7d21793e4bd85e", "size": 1143216, "path": "media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4"}, "eval/num_success": 5.0, "episode": 60.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "9ed1386a144a7cf88bb2e9788e0605909d7c93931a950927efa799acd16ddbc7", "size": 1131224, "path": "media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "bbf2275ce0ac22627299044cd7560c9382c060d0d20bbac7a9f9711f54e335e8", "size": 1151672, "path": "media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "1992567de3417dc1496f4d125be64105835ae9f18a5e8e053a7a2afbd6ddce74", "size": 1132300, "path": "media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://wes1tp6hv8mhyftf9eb2wllk8fkobsck6q14p8x08lc17fjqqeu9zy3rb4awjaznchz9665vmbdllyb4geip2yfmkpayl9of4nf6nh49jxipfvtzihjpleud21li38vh/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://t39nmjr785nbigkxn9jnai9fkk1sla2mx63ejhue63mwp7lnzk7fzb3oqm60yznnwcp69y5x5n34opdpl7urucfprriqduiieqdoqd0yx3rliwq247ygiqybwmhg635i:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 60, "total_num_episodes": 60, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9982039629416646, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.9455429463687541, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9530260766881642, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "bd3da959a3b78ffc3efce466157919356ff9c9199178d05be2a4dcb1b5896568", "size": 420, "artifact_path": "wandb-client-artifact://8mt2fkl4aicaeaiqlr30gcrniist5jedtez4geh5vyfrjdil3fwvqmtt2nwjppy3y8f2xnibmqtolf6up7k85h7wm0800w4ltiz4ppi2jpiak5a8okw9z2m2o2q9856f/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://a7x7eu8kkbzzpskl9n3tp8umw6iuscr2ijvg6i78fwq7tgs4u6v49860wz31l5j2nm7k8a14suumrbrj7ubd83cvmbz7lii8b8ncrtv8d30wwown2xh9x4uvqjy4i8eo:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 259}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9982039332389832, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.9455429315567017, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9530260562896729, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug-internal.log b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a8bd1f9084e2aea0690041408791739857ebac4f --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug-internal.log @@ -0,0 +1,760 @@ +2025-01-07 20:49:54,251 INFO StreamThr :2231710 [internal.py:wandb_internal():85] W&B internal server running at pid: 2231710, started at: 2025-01-07 20:49:54.251317 +2025-01-07 20:49:54,252 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:49:54,266 INFO WriterThread:2231710 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/run-o9qcewu3.wandb +2025-01-07 20:49:54,266 DEBUG SenderThread:2231710 [sender.py:send():379] send: header +2025-01-07 20:49:54,274 DEBUG SenderThread:2231710 [sender.py:send():379] send: run +2025-01-07 20:49:54,555 INFO SenderThread:2231710 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files +2025-01-07 20:49:54,555 INFO SenderThread:2231710 [sender.py:_start_run_threads():1188] run started: o9qcewu3 with start time 1736300994.25195 +2025-01-07 20:49:54,570 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:49:54,570 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:49:54,646 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:49:54,651 DEBUG HandlerThread:2231710 [system_info.py:__init__():26] System info init +2025-01-07 20:49:54,651 DEBUG HandlerThread:2231710 [system_info.py:__init__():41] System info init done +2025-01-07 20:49:54,651 INFO HandlerThread:2231710 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:49:54,651 INFO SystemMonitor:2231710 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:49:54,652 INFO HandlerThread:2231710 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:49:54,652 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:49:54,653 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:49:54,653 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:49:54,654 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:49:54,655 INFO SystemMonitor:2231710 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:49:54,664 DEBUG HandlerThread:2231710 [system_info.py:probe():152] Probing system +2025-01-07 20:49:54,666 DEBUG HandlerThread:2231710 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:49:54,671 DEBUG HandlerThread:2231710 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:49:54,671 DEBUG HandlerThread:2231710 [system_info.py:probe():200] Probing system done +2025-01-07 20:49:54,671 DEBUG HandlerThread:2231710 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:49:54.665027', 'startedAt': '2025-01-08T01:49:54.237296', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.7419999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.923, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 613.4297180175781}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:49:54,671 INFO HandlerThread:2231710 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:49:54,671 INFO HandlerThread:2231710 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:49:54,671 DEBUG HandlerThread:2231710 [system_info.py:_save_code():44] Saving code +2025-01-07 20:49:54,672 DEBUG HandlerThread:2231710 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:49:54,673 DEBUG HandlerThread:2231710 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:49:54,740 DEBUG HandlerThread:2231710 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:49:54,740 INFO HandlerThread:2231710 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:49:54,743 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:49:54,743 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:49:54,743 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:49:54,744 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:49:54,996 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/91k8dpks-wandb-metadata.json +2025-01-07 20:49:55,013 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:49:55,013 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:49:55,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:49:55,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:49:55,033 INFO wandb-upload_2:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/bhe3gypl-diff.patch +2025-01-07 20:49:55,036 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/3z9wb4zl-code/FISH/eval_robot.py +2025-01-07 20:49:55,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:55,057 INFO HandlerThread:2231710 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/tb" + save: true +} +control { + relay_id: "b5f985b5bb4a435d9f6388b1eebaee97" +} +_info { + stream_id: "o9qcewu3" +} + +2025-01-07 20:49:55,057 DEBUG HandlerThread:2231710 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:49:55,092 DEBUG SenderThread:2231710 [sender.py:send():379] send: telemetry +2025-01-07 20:49:55,108 DEBUG SenderThread:2231710 [sender.py:send():379] send: tbrecord +2025-01-07 20:49:55,109 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:49:55,109 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 with policy live +2025-01-07 20:49:55,283 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/g7dvd66t-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code/FISH/eval_robot.py +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/requirements.txt +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/diff.patch +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:49:55,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-metadata.json +2025-01-07 20:49:55,558 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code +2025-01-07 20:49:55,558 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code/FISH +2025-01-07 20:49:56,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:57,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:57,556 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:49:58,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:49:59,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:00,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:00,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:00,512 DEBUG SenderThread:2231710 [sender.py:send():379] send: config +2025-01-07 20:50:01,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:01,557 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:02,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:03,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:03,558 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:04,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:05,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:05,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:05,558 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:06,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:07,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:07,559 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:08,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:09,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:09,560 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:10,013 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:50:10,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:50:10,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:10,168 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:11,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:12,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:13,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:14,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:15,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:15,561 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:16,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:16,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:17,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:18,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:19,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:19,562 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:20,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:21,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:21,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:22,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:23,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:24,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:25,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:50:25,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:50:25,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:26,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:26,157 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:26,564 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml +2025-01-07 20:50:27,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:28,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:29,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:30,020 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:31,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:32,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:32,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:33,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:34,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:35,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:36,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:37,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:37,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:38,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:39,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:40,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:50:40,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:50:40,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:41,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:42,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:42,123 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:43,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:44,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:45,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:45,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:50:45,118 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:50:45,118 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 with policy now +2025-01-07 20:50:45,118 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:50:45,119 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:50:45,120 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:50:45,337 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/x2ibtcl5-media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 +2025-01-07 20:50:45,569 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:45,569 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:50:45,569 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 +2025-01-07 20:50:45,569 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:50:45,570 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos +2025-01-07 20:50:45,570 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media +2025-01-07 20:50:45,849 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/lizyi1pp-media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 +2025-01-07 20:50:46,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:47,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:47,570 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:48,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:48,120 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:49,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:50,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:51,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:51,571 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:52,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:53,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:53,572 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:53,610 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:54,023 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:54,655 DEBUG SystemMonitor:2231710 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:50:54,656 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:50:55,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:50:55,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:50:55,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:56,024 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:56,573 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:50:56,818 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/k72x4l9r-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:50:57,024 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:57,573 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:50:58,024 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:59,024 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:50:59,193 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:50:59,574 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:00,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:01,039 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:01,575 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:02,039 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:03,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:04,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:05,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:05,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:06,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:07,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:08,040 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:09,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:10,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:51:10,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:51:10,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:10,127 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:11,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:12,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:13,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:14,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:15,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:16,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:16,041 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:17,042 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:18,042 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:19,042 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:20,042 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:21,043 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:22,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:22,045 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:23,045 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:24,045 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:24,656 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:51:25,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:51:25,014 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:51:25,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:26,045 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:27,046 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:27,614 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:27,851 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:51:27,851 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 with policy now +2025-01-07 20:51:27,855 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:51:27,860 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:51:27,860 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:51:27,860 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:51:28,047 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:28,098 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/99n9bpv7-media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 +2025-01-07 20:51:28,589 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:51:28,589 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 +2025-01-07 20:51:28,590 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:51:28,864 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/ex6vn3j5-media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 +2025-01-07 20:51:29,050 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:29,589 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:30,050 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:31,050 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:32,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:32,645 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:33,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:33,591 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:34,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:35,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:35,592 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:36,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:37,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:38,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:38,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:39,049 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:40,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:51:40,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:51:40,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:41,050 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:42,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:42,462 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:51:42,462 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:51:42,462 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:51:42,594 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:51:42,594 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:51:43,021 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:43,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:43,594 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:44,052 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:45,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:45,595 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:46,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:47,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:47,596 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:51:48,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:48,919 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:49,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:50,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:51,051 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:52,053 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:53,053 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:54,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:51:54,053 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:54,656 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:51:55,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:51:55,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:51:55,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:56,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:57,053 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:58,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:59,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:51:59,157 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:00,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:01,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:02,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:03,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:04,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:05,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:05,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:06,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:07,055 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:08,054 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:09,055 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:10,014 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:52:10,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:52:10,057 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:10,090 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:11,055 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:12,055 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:13,059 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:13,604 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:14,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:14,197 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:52:14,197 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 with policy now +2025-01-07 20:52:14,200 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:52:14,201 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:52:14,201 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:52:14,201 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:52:14,416 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/azqrmhf1-media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 +2025-01-07 20:52:14,604 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:52:14,605 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 +2025-01-07 20:52:14,605 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:52:14,862 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/u2ogjcly-media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 +2025-01-07 20:52:15,061 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:15,605 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:16,028 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:16,063 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:16,605 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:17,064 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:18,077 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:19,066 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:19,606 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:20,069 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:21,064 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:21,885 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:22,065 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:23,066 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:23,607 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:24,065 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:24,657 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:52:25,019 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:52:25,020 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:52:25,067 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:26,072 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:27,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:27,172 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:28,072 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:28,609 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:52:28,850 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/xrx1rliz-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:52:29,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:29,298 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:52:29,298 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:52:29,299 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:52:29,609 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:52:29,610 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:52:29,610 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:30,081 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:31,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:31,610 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:32,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:32,428 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:33,073 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:33,611 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:34,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:35,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:35,612 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:36,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:37,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:38,026 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:38,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:39,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:40,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:52:40,018 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:52:40,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:41,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:42,074 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:43,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:43,138 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:44,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:45,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:46,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:47,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:48,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:48,685 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:49,075 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:49,616 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:50,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:51,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:51,617 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:52,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:53,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:53,618 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:52:53,897 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:52:54,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:54,657 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:52:55,015 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:52:55,015 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:52:55,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:56,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:57,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:58,076 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:59,077 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:52:59,170 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:00,080 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:01,097 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:01,183 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:53:01,183 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 with policy now +2025-01-07 20:53:01,190 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:53:01,191 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:53:01,191 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:53:01,192 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:53:01,397 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/ptvy46ax-media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 +2025-01-07 20:53:01,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:53:01,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:01,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 +2025-01-07 20:53:01,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:53:01,878 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/shg99opq-media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 +2025-01-07 20:53:02,099 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:02,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:03,107 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:03,621 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:04,103 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:04,205 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:05,105 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:06,108 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:07,110 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:07,623 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:08,106 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:09,107 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:09,623 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:09,845 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:10,017 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:53:10,017 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:53:10,108 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:11,111 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:12,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:13,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:14,109 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:15,033 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:15,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:16,112 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:17,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:17,367 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:53:17,367 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:53:17,368 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:53:17,626 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:53:17,626 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:53:18,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:19,120 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:19,627 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:20,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:20,847 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:21,112 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:22,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:23,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:23,628 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:24,113 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:24,658 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:53:25,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:53:25,016 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:53:25,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:26,100 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:26,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:27,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:28,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:29,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:30,114 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:31,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:32,031 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:32,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:33,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:34,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:35,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:36,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:37,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:38,031 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:38,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:39,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:40,016 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:53:40,017 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:53:40,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:41,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:42,116 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:43,080 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:43,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:44,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:45,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:46,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:47,117 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:48,094 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:48,118 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:49,117 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:53:49,117 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 with policy now +2025-01-07 20:53:49,118 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:53:49,118 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:53:49,118 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:53:49,121 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:53:49,125 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:49,327 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/rkjgfxwo-media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 +2025-01-07 20:53:49,640 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:53:49,641 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:49,641 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 +2025-01-07 20:53:49,641 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval +2025-01-07 20:53:49,946 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/afmz54iy-media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 +2025-01-07 20:53:50,120 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:51,126 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:51,641 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:52,122 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:53,124 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:54,009 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:53:54,133 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:54,658 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:53:55,022 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:53:55,022 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:53:55,126 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:55,643 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:56,187 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:57,195 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:57,643 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:53:58,191 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:59,195 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:53:59,853 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:00,192 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:01,195 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:02,199 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:03,207 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:04,197 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:04,428 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:54:04,428 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:04,429 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:04,595 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:54:04,595 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media +2025-01-07 20:54:04,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval +2025-01-07 20:54:04,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table +2025-01-07 20:54:04,723 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:54:04,826 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/4epar6xy-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:54:04,920 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/b057mig5-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:05,115 INFO wandb-upload_2:2231710 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvskn9n77 +2025-01-07 20:54:05,163 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:54:05,171 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:54:05,197 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:05,533 INFO SenderThread:2231710 [sender.py:send_request_log_artifact():1525] logged artifact run-o9qcewu3-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQyNTk3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0Mjg0MQ==', 'latestArtifact': None}} +2025-01-07 20:54:05,533 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:54:05,533 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 20:54:05,533 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:05,534 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:54:05,534 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,534 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,535 DEBUG SenderThread:2231710 [sender.py:send():379] send: telemetry +2025-01-07 20:54:05,535 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,539 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,539 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,539 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,539 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,540 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,540 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,540 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,540 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,540 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,541 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,541 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,541 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,541 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,542 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,542 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,542 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,542 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,543 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,543 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,543 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:05,544 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:05,544 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:54:05,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:05,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:05,646 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:05,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json +2025-01-07 20:54:05,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval +2025-01-07 20:54:05,781 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/p2qyvtdm-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:54:05,823 INFO wandb-upload_2:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/6fmddz6w-media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json +2025-01-07 20:54:05,930 INFO wandb-upload_1:2231710 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzwgvpbvm +2025-01-07 20:54:06,197 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:06,343 INFO SenderThread:2231710 [sender.py:send_request_log_artifact():1525] logged artifact run-o9qcewu3-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQyNTk5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0Mjg0Ng==', 'latestArtifact': None}} +2025-01-07 20:54:06,344 DEBUG SenderThread:2231710 [sender.py:send():379] send: files +2025-01-07 20:54:06,344 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json with policy now +2025-01-07 20:54:06,344 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:54:06,344 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:06,345 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:06,588 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/ga9zu2ca-media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json +2025-01-07 20:54:06,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:07,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:07,647 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:08,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:09,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:10,018 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:54:10,018 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:54:10,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:11,173 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:11,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:12,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:13,198 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:13,665 DEBUG SenderThread:2231710 [sender.py:send():379] send: exit +2025-01-07 20:54:13,665 INFO SenderThread:2231710 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:send_exit():588] handling runtime: 259 +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:send_exit():594] send defer +2025-01-07 20:54:13,666 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,666 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:54:13,666 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:54:13,666 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:54:13,667 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,667 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:54:13,667 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:13,667 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:54:13,667 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:54:13,667 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,667 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:54:13,667 INFO HandlerThread:2231710 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:54:13,667 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:54:13,667 DEBUG SystemMonitor:2231710 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:54:13,668 DEBUG SystemMonitor:2231710 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:54:13,668 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:54:13,720 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:54:13,720 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:54:13,720 INFO HandlerThread:2231710 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:54:13,721 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:13,721 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:54:13,721 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:54:13,721 DEBUG SenderThread:2231710 [sender.py:send():379] send: stats +2025-01-07 20:54:13,721 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,721 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:54:13,721 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:13,722 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:54:13,722 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:54:13,722 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:13,722 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:54:14,650 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:15,650 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:17,651 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml +2025-01-07 20:54:20,435 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:20,435 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:20,435 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:54:20,437 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:20,437 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:54:20,437 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:54:20,437 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:20,437 DEBUG SenderThread:2231710 [sender.py:send():379] send: history +2025-01-07 20:54:20,437 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:54:20,437 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:20,437 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:54:20,438 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:20,438 DEBUG SenderThread:2231710 [sender.py:send():379] send: summary +2025-01-07 20:54:20,441 INFO SenderThread:2231710 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:54:20,441 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:20,441 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:54:20,441 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:54:20,442 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:20,442 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:54:20,442 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:20,442 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:54:20,442 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:54:20,442 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:20,442 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:20,442 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:54:20,442 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:20,443 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:54:20,652 INFO Thread-12 :2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:20,667 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:22,095 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:54:22,095 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:22,095 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:22,096 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:54:22,096 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:22,096 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:54:22,096 INFO SenderThread:2231710 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:54:22,099 INFO SenderThread:2231710 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:54:22,101 INFO SenderThread:2231710 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:54:22,114 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:54:22,115 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:22,115 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:54:22,115 DEBUG SenderThread:2231710 [sender.py:send():379] send: artifact +2025-01-07 20:54:22,311 INFO SenderThread:2231710 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:54:22,311 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:22,311 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:54:22,311 INFO SenderThread:2231710 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:54:22,653 INFO SenderThread:2231710 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:22,653 INFO SenderThread:2231710 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml config.yaml +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json wandb-summary.json +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/requirements.txt requirements.txt +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:54:22,654 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log output.log +2025-01-07 20:54:22,659 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/diff.patch diff.patch +2025-01-07 20:54:22,659 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:22,660 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:54:22,661 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 media/videos/eval/0_eval_0_0760969f5acfd52720a3.mp4 +2025-01-07 20:54:22,661 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 media/videos/eval/3_eval_3_bbf2275ce0ac22627299.mp4 +2025-01-07 20:54:22,661 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 media/videos/eval/1_eval_1_ee6a2c2926ca8c08bde5.mp4 +2025-01-07 20:54:22,661 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 media/videos/eval/4_eval_4_1992567de3417dc1496f.mp4 +2025-01-07 20:54:22,662 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 media/videos/eval/2_eval_2_9ed1386a144a7cf88bb2.mp4 +2025-01-07 20:54:22,662 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json media/table/eval/total_success_rate_ci_6_bd3da959a3b78ffc3efc.table.json +2025-01-07 20:54:22,662 INFO SenderThread:2231710 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 20:54:22,663 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:54:22,663 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:22,663 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:54:22,663 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:22,663 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:54:22,663 INFO SenderThread:2231710 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:54:22,668 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:22,668 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:22,843 INFO wandb-upload_0:2231710 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/requirements.txt +2025-01-07 20:54:22,847 INFO wandb-upload_2:2231710 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/wandb-summary.json +2025-01-07 20:54:22,852 INFO wandb-upload_1:2231710 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/config.yaml +2025-01-07 20:54:22,879 INFO wandb-upload_3:2231710 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/files/output.log +2025-01-07 20:54:22,938 INFO wandb-upload_4:2231710 [upload_job.py:push():130] Uploaded file /tmp/tmp8t8glpqfwandb/k6ad1g9x-events.out.tfevents.1736300995.leonmkim-ROG-Strix-G15CS-G15CS.2231543.0 +2025-01-07 20:54:23,139 INFO Thread-11 (_thread_body):2231710 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:54:23,139 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:23,139 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:54:23,139 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:23,139 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:54:23,139 INFO SenderThread:2231710 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:54:23,140 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:54:23,141 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:23,141 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:54:23,141 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:23,141 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:54:23,141 INFO SenderThread:2231710 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:54:23,364 INFO SenderThread:2231710 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:54:23,365 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:54:23,365 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:23,365 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:54:23,365 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:23,365 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:54:23,365 INFO SenderThread:2231710 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:54:23,365 DEBUG SenderThread:2231710 [sender.py:send():379] send: final +2025-01-07 20:54:23,365 DEBUG SenderThread:2231710 [sender.py:send():379] send: footer +2025-01-07 20:54:23,365 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:54:23,365 INFO HandlerThread:2231710 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:54:23,366 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: defer +2025-01-07 20:54:23,366 INFO SenderThread:2231710 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:54:23,366 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:23,366 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:54:23,367 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:54:23,367 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:54:23,367 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:23,367 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:54:23,368 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:54:23,368 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:23,436 DEBUG SenderThread:2231710 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:54:23,436 INFO MainThread:2231710 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:54:23,437 INFO MainThread:2231710 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:54:23,437 INFO MainThread:2231710 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:54:23,437 DEBUG HandlerThread:2231710 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:54:23,437 INFO HandlerThread:2231710 [handler.py:finish():882] shutting down handler +2025-01-07 20:54:24,367 INFO WriterThread:2231710 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/run-o9qcewu3.wandb +2025-01-07 20:54:24,436 INFO SenderThread:2231710 [sender.py:finish():1615] shutting down sender +2025-01-07 20:54:24,436 INFO SenderThread:2231710 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:54:24,436 INFO SenderThread:2231710 [file_pusher.py:join():175] waiting for file pusher diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug.log b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..97f81e56e3ba0ca3d3cb38f32ffbe7c18f0a03e0 --- /dev/null +++ b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Configure stats pid to 2231543 +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/settings +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug.log +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/logs/debug-internal.log +2025-01-07 20:49:54,243 INFO MainThread:2231543 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:49:54,244 INFO MainThread:2231543 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3'} +2025-01-07 20:49:54,244 INFO MainThread:2231543 [wandb_init.py:init():619] starting backend +2025-01-07 20:49:54,244 INFO MainThread:2231543 [wandb_init.py:init():623] setting up manager +2025-01-07 20:49:54,250 INFO MainThread:2231543 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:49:54,251 INFO MainThread:2231543 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:49:54,266 INFO MainThread:2231543 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:49:54,272 INFO MainThread:2231543 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:49:54,569 INFO MainThread:2231543 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:49:54,641 INFO MainThread:2231543 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:49:54,641 INFO MainThread:2231543 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:49:55,015 INFO MainThread:2231543 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:49:55,015 INFO MainThread:2231543 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:49:55,015 INFO MainThread:2231543 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:49:55,016 INFO MainThread:2231543 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:49:55,018 INFO MainThread:2231543 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:49:55,018 INFO MainThread:2231543 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/o9qcewu3/tb, True +2025-01-07 20:50:00,511 INFO MainThread:2231543 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:54:24,438 WARNING MsgRouterThr:2231543 [router.py:message_loop():77] message_loop has been closed diff --git a/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/run-o9qcewu3.wandb b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/run-o9qcewu3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4fa574b150f0afcd5a4e0d9bde76667463ff9f34 Binary files /dev/null and b/o9qcewu3/wandb/run-20250107_204954-o9qcewu3/run-o9qcewu3.wandb differ diff --git a/t3rrx672/.hydra/config.yaml b/t3rrx672/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab --- /dev/null +++ b/t3rrx672/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1009_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/t3rrx672/.hydra/hydra.yaml b/t3rrx672/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a45b391202fd471c76bd6e23e01365560ab0a0ac --- /dev/null +++ b/t3rrx672/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/t3rrx672 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/t3rrx672/.hydra/overrides.yaml b/t3rrx672/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/t3rrx672/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/t3rrx672/eval_policy.log b/t3rrx672/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..61634fce578a6f9ab17b9877d2e088abee048c07 --- /dev/null +++ b/t3rrx672/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 17:09:07,248][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:09:07,251][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 17:09:08,591][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/w3dz1sd1/episode_rosbags/episode_3_2025-01-07-17-48-43.bag b/w3dz1sd1/episode_rosbags/episode_3_2025-01-07-17-48-43.bag new file mode 100644 index 0000000000000000000000000000000000000000..e612bdaf9a4d02edfe4596ebd8914fb60f362993 --- /dev/null +++ b/w3dz1sd1/episode_rosbags/episode_3_2025-01-07-17-48-43.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c15cbfadc252ae8790fe010fc753695c32418f1bfc287851b1d4f2e59f651c99 +size 1408072149 diff --git a/w3dz1sd1/episode_rosbags/episode_4_2025-01-07-17-49-25.bag b/w3dz1sd1/episode_rosbags/episode_4_2025-01-07-17-49-25.bag new file mode 100644 index 0000000000000000000000000000000000000000..f3fdd169afa6871ccb760a509e02dc09e2d31660 --- /dev/null +++ b/w3dz1sd1/episode_rosbags/episode_4_2025-01-07-17-49-25.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9465d73485ae7bef95de301d44215a167c84c09bfef8c6b4fbfb04885ca96d6 +size 1410907384